FP32 requantization in QS8 GEMM/IGEMM microkernels for SSE/AVX/XOP
PiperOrigin-RevId: 376966195
diff --git a/BUILD.bazel b/BUILD.bazel
index 2dfa47a..ae87547 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -2620,41 +2620,69 @@
"src/qs8-gavgpool/gen/7x-minmax-sse2-c8-acc2.c",
"src/qs8-gavgpool/gen/7x-minmax-sse2-c16-acc2.c",
"src/qs8-gavgpool/gen/7x-minmax-sse2-c24-acc2.c",
- "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c",
"src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c",
"src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c",
- "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c",
"src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c",
"src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c",
- "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c",
"src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c",
"src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c",
- "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c",
"src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c",
"src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c",
- "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c",
"src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c",
"src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c",
- "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c",
"src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c",
"src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c",
- "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c",
"src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c",
"src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c",
- "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c",
"src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c",
- "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c",
"src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c",
- "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c",
"src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c",
- "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c",
"src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c",
- "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c",
"src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c",
- "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c",
"src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c",
- "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c",
"src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c",
"src/qs8-requantization/fp32-sse2.c",
"src/qs8-requantization/gemmlowp-sse2.c",
"src/qs8-requantization/rndna-sse2.c",
@@ -2715,41 +2743,69 @@
"src/qs8-gavgpool/gen/7x-minmax-ssse3-c8-acc2.c",
"src/qs8-gavgpool/gen/7x-minmax-ssse3-c16-acc2.c",
"src/qs8-gavgpool/gen/7x-minmax-ssse3-c24-acc2.c",
- "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c",
"src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c",
"src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c",
- "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c",
"src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c",
"src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c",
- "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c",
"src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c",
"src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c",
- "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c",
"src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c",
"src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c",
- "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c",
"src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c",
"src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c",
- "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c",
"src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c",
"src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c",
- "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c",
"src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c",
"src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c",
- "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c",
"src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c",
- "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c",
"src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c",
- "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c",
"src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c",
- "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c",
"src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c",
- "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c",
"src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c",
- "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c",
"src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c",
- "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c",
"src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c",
"src/qs8-requantization/gemmlowp-ssse3.c",
"src/qs8-requantization/rndna-ssse3.c",
"src/qu8-requantization/gemmlowp-ssse3.c",
@@ -2815,41 +2871,69 @@
"src/qs8-gavgpool/gen/7x-minmax-sse41-c8-acc2.c",
"src/qs8-gavgpool/gen/7x-minmax-sse41-c16-acc2.c",
"src/qs8-gavgpool/gen/7x-minmax-sse41-c24-acc2.c",
- "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c",
"src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c",
"src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c",
- "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c",
"src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c",
"src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c",
- "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c",
"src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c",
"src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c",
- "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c",
"src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c",
"src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c",
- "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c",
"src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c",
"src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c",
- "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c",
"src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c",
"src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c",
- "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c",
"src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c",
"src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c",
- "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c",
"src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c",
- "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c",
"src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c",
- "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c",
"src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c",
- "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c",
"src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c",
- "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c",
"src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c",
- "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c",
"src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c",
- "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c",
"src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c",
"src/qs8-requantization/fp32-sse4.c",
"src/qs8-requantization/gemmlowp-sse4.c",
"src/qs8-requantization/rndna-sse4.c",
@@ -3032,41 +3116,69 @@
"src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c",
"src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c",
"src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c",
- "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c",
"src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c",
"src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c",
- "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c",
"src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c",
"src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c",
- "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c",
"src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c",
"src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c",
- "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c",
"src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c",
"src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c",
- "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c",
"src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c",
"src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c",
- "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c",
"src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c",
"src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c",
- "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c",
"src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c",
"src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c",
- "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c",
"src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c",
- "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c",
"src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c",
- "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c",
"src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c",
- "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c",
"src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c",
- "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c",
"src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c",
- "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c",
"src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c",
- "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c",
"src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c",
"src/qs8-vadd/gen/minmax-avx-mul16-ld64-x8.c",
"src/qs8-vadd/gen/minmax-avx-mul16-ld64-x16.c",
"src/qs8-vadd/gen/minmax-avx-mul16-ld64-x24.c",
@@ -3092,41 +3204,69 @@
"src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c",
"src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c",
"src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c",
- "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c",
"src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c",
+ "src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c",
"src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c",
- "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c",
"src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c",
+ "src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c",
"src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c",
- "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c",
"src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c",
+ "src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c",
"src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c",
- "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c",
"src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c",
+ "src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c",
"src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c",
- "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c",
"src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c",
+ "src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c",
"src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c",
- "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c",
"src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c",
+ "src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c",
"src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c",
- "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c",
"src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c",
+ "src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c",
"src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c",
- "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c",
"src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c",
- "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c",
"src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c",
- "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c",
"src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c",
- "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c",
"src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c",
- "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c",
"src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c",
- "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c",
"src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c",
- "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c",
"src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c",
+ "src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c",
"src/qs8-vadd/gen/minmax-xop-mul32-ld32-x8.c",
"src/qs8-vadd/gen/minmax-xop-mul32-ld32-x16.c",
"src/qs8-vadd/gen/minmax-xop-mul32-ld32-x24.c",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 957ef47..becb880 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1860,41 +1860,69 @@
src/qs8-gavgpool/gen/7x-minmax-sse2-c8-acc2.c
src/qs8-gavgpool/gen/7x-minmax-sse2-c16-acc2.c
src/qs8-gavgpool/gen/7x-minmax-sse2-c24-acc2.c
- src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
- src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
- src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
- src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
- src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
- src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
- src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
- src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
src/qs8-requantization/fp32-sse2.c
src/qs8-requantization/gemmlowp-sse2.c
src/qs8-requantization/rndna-sse2.c
@@ -1954,41 +1982,69 @@
src/qs8-gavgpool/gen/7x-minmax-ssse3-c8-acc2.c
src/qs8-gavgpool/gen/7x-minmax-ssse3-c16-acc2.c
src/qs8-gavgpool/gen/7x-minmax-ssse3-c24-acc2.c
- src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
- src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
+ src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
- src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
+ src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
- src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
+ src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
- src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
+ src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
- src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+ src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
- src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+ src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
- src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
- src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
- src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
- src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
- src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
- src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
- src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
src/qs8-requantization/gemmlowp-ssse3.c
src/qs8-requantization/rndna-ssse3.c
src/qu8-requantization/gemmlowp-ssse3.c
@@ -2053,41 +2109,69 @@
src/qs8-gavgpool/gen/7x-minmax-sse41-c8-acc2.c
src/qs8-gavgpool/gen/7x-minmax-sse41-c16-acc2.c
src/qs8-gavgpool/gen/7x-minmax-sse41-c24-acc2.c
- src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
- src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
+ src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
- src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
+ src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
- src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
+ src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
- src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
+ src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
- src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+ src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
- src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+ src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
- src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
- src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
- src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
- src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
- src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
- src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
- src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
src/qs8-requantization/fp32-sse4.c
src/qs8-requantization/gemmlowp-sse4.c
src/qs8-requantization/rndna-sse4.c
@@ -2267,41 +2351,69 @@
src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c
src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c
src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c
- src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
- src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
+ src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
- src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
+ src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
- src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
+ src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
- src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
+ src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
- src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+ src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
- src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+ src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
- src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
- src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
- src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
- src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
- src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
- src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
- src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
src/qs8-vadd/gen/minmax-avx-mul16-ld64-x8.c
src/qs8-vadd/gen/minmax-avx-mul16-ld64-x16.c
src/qs8-vadd/gen/minmax-avx-mul16-ld64-x24.c
@@ -2326,41 +2438,69 @@
src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c
src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c
src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c
- src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
+ src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
- src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
+ src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
+ src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
- src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
+ src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
+ src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
- src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
+ src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
+ src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
- src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
+ src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
+ src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
- src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
+ src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+ src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
- src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
+ src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+ src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
- src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
+ src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
- src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
+ src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
- src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
+ src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
- src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
+ src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
- src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
+ src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
- src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
+ src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
- src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+ src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
src/qs8-vadd/gen/minmax-xop-mul32-ld32-x8.c
src/qs8-vadd/gen/minmax-xop-mul32-ld32-x16.c
src/qs8-vadd/gen/minmax-xop-mul32-ld32-x24.c
diff --git a/bench/qs8-gemm-e2e.cc b/bench/qs8-gemm-e2e.cc
index 8d5d319..8cf6b45 100644
--- a/bench/qs8-gemm-e2e.cc
+++ b/bench/qs8-gemm-e2e.cc
@@ -1399,6 +1399,16 @@
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_1x4c2_fp32__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_1x4c2_gemmlowp__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1410,6 +1420,16 @@
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_1x4c2_fp32__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c2_gemmlowp__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -1422,6 +1442,16 @@
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_2x4c2_fp32__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_2x4c2_gemmlowp__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1433,6 +1463,16 @@
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_2x4c2_fp32__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_3x4c2_gemmlowp__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1444,6 +1484,16 @@
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_3x4c2_fp32__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_3x4c2_gemmlowp__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1455,6 +1505,16 @@
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_3x4c2_fp32__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_4x4c2_gemmlowp__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1466,6 +1526,16 @@
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_4x4c2_fp32__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_4x4c2_gemmlowp__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1477,6 +1547,16 @@
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_4x4c2_fp32__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -1490,6 +1570,16 @@
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_1x4c8_fp32__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_1x4c8_gemmlowp__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1501,6 +1591,16 @@
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_1x4c8_fp32__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c8_gemmlowp__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -1513,6 +1613,16 @@
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_2x4c8_fp32__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_2x4c8_gemmlowp__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1524,6 +1634,16 @@
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_2x4c8_fp32__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_3x4c8_gemmlowp__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1535,6 +1655,16 @@
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_3x4c8_fp32__xop_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_3x4c8_gemmlowp__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1546,6 +1676,16 @@
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_3x4c8_fp32__xop_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckXOP);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -1559,6 +1699,16 @@
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_1x4c2_fp32__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_1x4c2_gemmlowp__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1570,6 +1720,16 @@
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_1x4c2_fp32__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c2_gemmlowp__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -1582,6 +1742,16 @@
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_2x4c2_fp32__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_2x4c2_gemmlowp__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1593,6 +1763,16 @@
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_2x4c2_fp32__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_3x4c2_gemmlowp__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1604,6 +1784,16 @@
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_3x4c2_fp32__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_3x4c2_gemmlowp__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1615,6 +1805,16 @@
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_3x4c2_fp32__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_4x4c2_gemmlowp__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1626,6 +1826,16 @@
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_4x4c2_fp32__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_4x4c2_gemmlowp__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1637,6 +1847,16 @@
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_4x4c2_fp32__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -1650,6 +1870,16 @@
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_1x4c8_fp32__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_1x4c8_gemmlowp__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1661,6 +1891,16 @@
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_1x4c8_fp32__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c8_gemmlowp__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -1673,6 +1913,16 @@
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_2x4c8_fp32__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_2x4c8_gemmlowp__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1684,6 +1934,16 @@
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_2x4c8_fp32__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_3x4c8_gemmlowp__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1695,6 +1955,16 @@
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_3x4c8_fp32__avx_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_3x4c8_gemmlowp__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1706,6 +1976,16 @@
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_3x4c8_fp32__avx_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckAVX);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -1719,6 +1999,16 @@
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_1x4c2_fp32__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_1x4c2_gemmlowp__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1730,6 +2020,16 @@
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_1x4c2_fp32__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c2_gemmlowp__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -1742,6 +2042,16 @@
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_2x4c2_fp32__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_2x4c2_gemmlowp__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1753,6 +2063,16 @@
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_2x4c2_fp32__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_3x4c2_gemmlowp__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1764,6 +2084,16 @@
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_3x4c2_fp32__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_3x4c2_gemmlowp__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1775,6 +2105,16 @@
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_3x4c2_fp32__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_4x4c2_gemmlowp__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1786,6 +2126,16 @@
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_4x4c2_fp32__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_4x4c2_gemmlowp__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1797,6 +2147,16 @@
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_4x4c2_fp32__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -1810,6 +2170,16 @@
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_1x4c8_fp32__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_1x4c8_gemmlowp__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1821,6 +2191,16 @@
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_1x4c8_fp32__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c8_gemmlowp__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -1833,6 +2213,16 @@
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_2x4c8_fp32__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_2x4c8_gemmlowp__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1844,6 +2234,16 @@
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_2x4c8_fp32__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_3x4c8_gemmlowp__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1855,6 +2255,16 @@
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_3x4c8_fp32__sse41_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_3x4c8_gemmlowp__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1866,6 +2276,16 @@
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_3x4c8_fp32__sse41_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSE41);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -1879,6 +2299,16 @@
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_1x4c2_fp32__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_1x4c2_gemmlowp__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1890,6 +2320,16 @@
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_1x4c2_fp32__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c2_gemmlowp__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -1902,6 +2342,16 @@
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_2x4c2_fp32__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_2x4c2_gemmlowp__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1913,6 +2363,16 @@
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_2x4c2_fp32__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_3x4c2_gemmlowp__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1924,6 +2384,16 @@
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_3x4c2_fp32__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_3x4c2_gemmlowp__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1935,6 +2405,16 @@
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_3x4c2_fp32__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_4x4c2_gemmlowp__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1946,6 +2426,16 @@
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_4x4c2_fp32__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_4x4c2_gemmlowp__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1957,6 +2447,16 @@
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_4x4c2_fp32__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -1970,6 +2470,16 @@
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_1x4c8_fp32__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_1x4c8_gemmlowp__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -1981,6 +2491,16 @@
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_1x4c8_fp32__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c8_gemmlowp__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -1993,6 +2513,16 @@
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_2x4c8_fp32__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_2x4c8_gemmlowp__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2004,6 +2534,16 @@
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_2x4c8_fp32__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_3x4c8_gemmlowp__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2015,6 +2555,16 @@
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_3x4c8_fp32__ssse3_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_3x4c8_gemmlowp__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2026,6 +2576,16 @@
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_3x4c8_fp32__ssse3_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */,
+ benchmark::utils::CheckSSSE3);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -2038,6 +2598,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_1x4c2_fp32__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_1x4c2_gemmlowp__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2048,6 +2617,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_1x4c2_fp32__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 1 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c2_gemmlowp__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -2059,6 +2637,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_2x4c2_fp32__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_2x4c2_gemmlowp__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2069,6 +2656,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_2x4c2_fp32__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 2 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_3x4c2_gemmlowp__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2079,6 +2675,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_3x4c2_fp32__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_3x4c2_gemmlowp__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2089,6 +2694,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_3x4c2_fp32__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 3 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_4x4c2_gemmlowp__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2099,6 +2713,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_4x4c2_fp32__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_4x4c2_gemmlowp__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2109,6 +2732,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_4x4c2_fp32__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 4 /* mr */, 4 /* nr */, 1 /* log2_kr */, 0 /* log2_sr */);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -2121,6 +2753,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_1x4c8_fp32__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_1x4c8_gemmlowp__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2131,6 +2772,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_1x4c8_fp32__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 1 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
+ }
#endif // XNN_ENABLE_FULL_BENCHMARKS
static void qs8_gemm_2x4c8_gemmlowp__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
@@ -2142,6 +2792,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_2x4c8_fp32__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_2x4c8_gemmlowp__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2152,6 +2811,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_2x4c8_fp32__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 2 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_3x4c8_gemmlowp__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2162,6 +2830,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_3x4c8_fp32__sse2_ld64(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
+ }
static void qs8_gemm_3x4c8_gemmlowp__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
GEMMEnd2EndBenchmark(state, model,
@@ -2172,6 +2849,15 @@
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params,
3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
}
+ static void qs8_gemm_3x4c8_fp32__sse2_ld128(benchmark::State& state, models::ExecutionPlanFactory model) {
+ GEMMEnd2EndBenchmark(state, model,
+ xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld128,
+ xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128,
+ xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld128,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params,
+ 3 /* mr */, 4 /* nr */, 3 /* log2_kr */, 0 /* log2_sr */);
+ }
#if XNN_ENABLE_FULL_BENCHMARKS
@@ -2196,103 +2882,173 @@
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__xop_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__xop_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__xop_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__xop_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__xop_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__xop_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__xop_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__xop_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__xop_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__xop_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__xop_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__xop_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__xop_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__xop_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__xop_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__xop_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__xop_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__xop_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__xop_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__xop_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__xop_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__xop_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__xop_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__xop_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__xop_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__xop_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__xop_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__xop_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__avx_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__avx_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__avx_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__avx_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__avx_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__avx_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__avx_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__avx_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__avx_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__avx_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__avx_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__avx_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__avx_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__avx_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__avx_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__avx_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__avx_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__avx_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__avx_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__avx_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__avx_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__avx_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__avx_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__avx_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__avx_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__avx_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__avx_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__avx_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__sse41_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__sse41_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__sse41_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__sse41_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__sse41_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__sse41_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__sse41_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__sse41_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__sse41_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__sse41_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__sse41_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__sse41_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__sse41_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__sse41_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__sse41_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__sse41_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__sse41_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__sse41_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__sse41_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__sse41_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__sse41_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__sse41_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__sse41_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__sse41_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__sse41_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__sse41_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__sse41_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__sse41_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__ssse3_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__ssse3_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__ssse3_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__ssse3_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__ssse3_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__ssse3_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__ssse3_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__ssse3_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__ssse3_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__ssse3_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__ssse3_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__ssse3_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__ssse3_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__ssse3_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__ssse3_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__ssse3_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__ssse3_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__ssse3_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__ssse3_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__ssse3_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__ssse3_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__ssse3_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__ssse3_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__ssse3_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__ssse3_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__ssse3_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__ssse3_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__ssse3_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__sse2_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__sse2_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_gemmlowp__sse2_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c2_fp32__sse2_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__sse2_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__sse2_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_gemmlowp__sse2_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c2_fp32__sse2_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__sse2_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__sse2_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_gemmlowp__sse2_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c2_fp32__sse2_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__sse2_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__sse2_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_gemmlowp__sse2_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_4x4c2_fp32__sse2_ld128);
#if XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__sse2_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__sse2_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_gemmlowp__sse2_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_1x4c8_fp32__sse2_ld128);
#endif // XNN_ENABLE_FULL_BENCHMARKS
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__sse2_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__sse2_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_gemmlowp__sse2_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_2x4c8_fp32__sse2_ld128);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__sse2_ld64);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__sse2_ld64);
BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_gemmlowp__sse2_ld128);
+ BENCHMARK_QS8_END2END(qs8_gemm_3x4c8_fp32__sse2_ld128);
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
#if XNN_ARCH_WASMSIMD
diff --git a/bench/qs8-gemm.cc b/bench/qs8-gemm.cc
index c5ac7f6..48c41ab 100644
--- a/bench/qs8-gemm.cc
+++ b/bench/qs8-gemm.cc
@@ -742,27 +742,51 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld64, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_2x4c2_fp32__xop_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_3x4c2_gemmlowp__xop_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld64, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_3x4c2_fp32__xop_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_4x4c2_gemmlowp__xop_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld64, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_4x4c2_fp32__xop_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_2x4c2_gemmlowp__xop_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld128, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_2x4c2_fp32__xop_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_3x4c2_gemmlowp__xop_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld128, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_3x4c2_fp32__xop_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_4x4c2_gemmlowp__xop_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld128, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_4x4c2_fp32__xop_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_xw_2x4c2_gemmlowp__xop(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c2__xop, 2, 4, 2, 1,
@@ -781,19 +805,35 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_2x4c8_fp32__xop_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_3x4c8_gemmlowp__xop_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld64, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_3x4c8_fp32__xop_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_2x4c8_gemmlowp__xop_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld128, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_2x4c8_fp32__xop_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_3x4c8_gemmlowp__xop_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld128, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckXOP);
}
+ static void qs8_gemm_3x4c8_fp32__xop_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckXOP);
+ }
static void qs8_gemm_xw_2x4c8_gemmlowp__xop(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c8__xop, 2, 4, 8, 1,
@@ -808,27 +848,51 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld64, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_2x4c2_fp32__avx_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_3x4c2_gemmlowp__avx_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld64, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_3x4c2_fp32__avx_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_4x4c2_gemmlowp__avx_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld64, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_4x4c2_fp32__avx_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_2x4c2_gemmlowp__avx_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld128, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_2x4c2_fp32__avx_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_3x4c2_gemmlowp__avx_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld128, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_3x4c2_fp32__avx_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_4x4c2_gemmlowp__avx_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld128, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_4x4c2_fp32__avx_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_xw_2x4c2_gemmlowp__avx(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c2__avx, 2, 4, 2, 1,
@@ -847,19 +911,35 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld64, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_2x4c8_fp32__avx_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_3x4c8_gemmlowp__avx_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld64, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_3x4c8_fp32__avx_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_2x4c8_gemmlowp__avx_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_2x4c8_fp32__avx_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_3x4c8_gemmlowp__avx_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld128, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckAVX);
}
+ static void qs8_gemm_3x4c8_fp32__avx_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckAVX);
+ }
static void qs8_gemm_xw_2x4c8_gemmlowp__avx(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c8__avx, 2, 4, 8, 1,
@@ -874,27 +954,51 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld64, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_2x4c2_fp32__sse41_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_3x4c2_gemmlowp__sse41_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld64, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_3x4c2_fp32__sse41_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_4x4c2_gemmlowp__sse41_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld64, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_4x4c2_fp32__sse41_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_2x4c2_gemmlowp__sse41_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld128, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_2x4c2_fp32__sse41_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_3x4c2_gemmlowp__sse41_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld128, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_3x4c2_fp32__sse41_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_4x4c2_gemmlowp__sse41_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld128, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_4x4c2_fp32__sse41_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_xw_2x4c2_gemmlowp__sse41(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c2__sse41, 2, 4, 2, 1,
@@ -913,19 +1017,35 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld64, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_2x4c8_fp32__sse41_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_3x4c8_gemmlowp__sse41_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_3x4c8_fp32__sse41_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_2x4c8_gemmlowp__sse41_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld128, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_2x4c8_fp32__sse41_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_3x4c8_gemmlowp__sse41_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld128, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse4_params, benchmark::utils::CheckSSE41);
}
+ static void qs8_gemm_3x4c8_fp32__sse41_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse4_params, benchmark::utils::CheckSSE41);
+ }
static void qs8_gemm_xw_2x4c8_gemmlowp__sse41(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c8__sse41, 2, 4, 8, 1,
@@ -940,27 +1060,51 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld64, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_2x4c2_fp32__ssse3_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_3x4c2_gemmlowp__ssse3_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld64, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_3x4c2_fp32__ssse3_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_4x4c2_gemmlowp__ssse3_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld64, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_4x4c2_fp32__ssse3_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_2x4c2_gemmlowp__ssse3_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld128, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_2x4c2_fp32__ssse3_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_3x4c2_gemmlowp__ssse3_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld128, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_3x4c2_fp32__ssse3_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_4x4c2_gemmlowp__ssse3_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld128, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_4x4c2_fp32__ssse3_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_xw_2x4c2_gemmlowp__ssse3(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c2__ssse3, 2, 4, 2, 1,
@@ -979,19 +1123,35 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld64, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_2x4c8_fp32__ssse3_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_3x4c8_gemmlowp__ssse3_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_3x4c8_fp32__ssse3_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_2x4c8_gemmlowp__ssse3_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld128, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_2x4c8_fp32__ssse3_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_3x4c8_gemmlowp__ssse3_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld128, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params, benchmark::utils::CheckSSSE3);
}
+ static void qs8_gemm_3x4c8_fp32__ssse3_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params, benchmark::utils::CheckSSSE3);
+ }
static void qs8_gemm_xw_2x4c8_gemmlowp__ssse3(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c8__ssse3, 2, 4, 8, 1,
@@ -1006,27 +1166,51 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse2_ld64, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_2x4c2_fp32__sse2_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_3x4c2_gemmlowp__sse2_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld64, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_3x4c2_fp32__sse2_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_4x4c2_gemmlowp__sse2_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld64, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_4x4c2_fp32__sse2_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_2x4c2_gemmlowp__sse2_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse2_ld128, 2, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_2x4c2_fp32__sse2_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, 2, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_3x4c2_gemmlowp__sse2_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld128, 3, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_3x4c2_fp32__sse2_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, 3, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_4x4c2_gemmlowp__sse2_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld128, 4, 4, 2, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_4x4c2_fp32__sse2_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, 4, 4, 2, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_xw_2x4c2_gemmlowp__sse2(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c2__sse2, 2, 4, 2, 1,
@@ -1045,19 +1229,35 @@
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld64, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_2x4c8_fp32__sse2_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_3x4c8_gemmlowp__sse2_ld64(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_3x4c8_fp32__sse2_ld64(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_2x4c8_gemmlowp__sse2_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld128, 2, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_2x4c8_fp32__sse2_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, 2, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_3x4c8_gemmlowp__sse2_ld128(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld128, 3, 4, 8, 1,
xnn_init_qs8_conv_minmax_gemmlowp_sse2_params);
}
+ static void qs8_gemm_3x4c8_fp32__sse2_ld128(benchmark::State& state, const char* net) {
+ GEMMBenchmark(state, xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, 3, 4, 8, 1,
+ xnn_init_qs8_conv_minmax_fp32_sse2_params);
+ }
static void qs8_gemm_xw_2x4c8_gemmlowp__sse2(benchmark::State& state, const char* net) {
GEMMBenchmark(state, xnn_qs8_gemm_xw_minmax_gemmlowp_ukernel_2x4c8__sse2, 2, 4, 8, 1,
@@ -1083,82 +1283,132 @@
BENCHMARK_GEMM(qs8_gemm_xw_3x8c8_gemmlowp__avx2)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__xop_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__xop_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__xop_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__xop_ld64)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__xop_ld64)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__xop_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__xop_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__xop_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__xop_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__xop_ld128)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__xop_ld128)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__xop_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c2_gemmlowp__xop)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c2_gemmlowp__xop)
BENCHMARK_GEMM(qs8_gemm_xw_4x4c2_gemmlowp__xop)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__xop_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__xop_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__xop_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__xop_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__xop_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__xop_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__xop_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__xop_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c8_gemmlowp__xop)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c8_gemmlowp__xop)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__avx_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__avx_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__avx_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__avx_ld64)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__avx_ld64)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__avx_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__avx_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__avx_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__avx_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__avx_ld128)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__avx_ld128)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__avx_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c2_gemmlowp__avx)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c2_gemmlowp__avx)
BENCHMARK_GEMM(qs8_gemm_xw_4x4c2_gemmlowp__avx)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__avx_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__avx_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__avx_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__avx_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__avx_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__avx_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__avx_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__avx_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c8_gemmlowp__avx)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c8_gemmlowp__avx)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__sse41_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__sse41_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__sse41_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__sse41_ld64)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__sse41_ld64)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__sse41_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__sse41_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__sse41_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__sse41_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__sse41_ld128)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__sse41_ld128)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__sse41_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c2_gemmlowp__sse41)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c2_gemmlowp__sse41)
BENCHMARK_GEMM(qs8_gemm_xw_4x4c2_gemmlowp__sse41)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__sse41_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__sse41_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__sse41_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__sse41_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__sse41_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__sse41_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__sse41_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__sse41_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c8_gemmlowp__sse41)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c8_gemmlowp__sse41)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__ssse3_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__ssse3_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__ssse3_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__ssse3_ld64)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__ssse3_ld64)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__ssse3_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__ssse3_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__ssse3_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__ssse3_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__ssse3_ld128)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__ssse3_ld128)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__ssse3_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c2_gemmlowp__ssse3)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c2_gemmlowp__ssse3)
BENCHMARK_GEMM(qs8_gemm_xw_4x4c2_gemmlowp__ssse3)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__ssse3_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__ssse3_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__ssse3_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__ssse3_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__ssse3_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__ssse3_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__ssse3_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__ssse3_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c8_gemmlowp__ssse3)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c8_gemmlowp__ssse3)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__sse2_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__sse2_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__sse2_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__sse2_ld64)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__sse2_ld64)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__sse2_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c2_gemmlowp__sse2_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c2_fp32__sse2_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c2_gemmlowp__sse2_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c2_fp32__sse2_ld128)
BENCHMARK_GEMM(qs8_gemm_4x4c2_gemmlowp__sse2_ld128)
+ BENCHMARK_GEMM(qs8_gemm_4x4c2_fp32__sse2_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c2_gemmlowp__sse2)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c2_gemmlowp__sse2)
BENCHMARK_GEMM(qs8_gemm_xw_4x4c2_gemmlowp__sse2)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__sse2_ld64)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__sse2_ld64)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__sse2_ld64)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__sse2_ld64)
BENCHMARK_GEMM(qs8_gemm_2x4c8_gemmlowp__sse2_ld128)
+ BENCHMARK_GEMM(qs8_gemm_2x4c8_fp32__sse2_ld128)
BENCHMARK_GEMM(qs8_gemm_3x4c8_gemmlowp__sse2_ld128)
+ BENCHMARK_GEMM(qs8_gemm_3x4c8_fp32__sse2_ld128)
BENCHMARK_GEMM(qs8_gemm_xw_2x4c8_gemmlowp__sse2)
BENCHMARK_GEMM(qs8_gemm_xw_3x4c8_gemmlowp__sse2)
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
diff --git a/scripts/generate-qs8-gemm.sh b/scripts/generate-qs8-gemm.sh
index 424376c..c0816ed 100755
--- a/scripts/generate-qs8-gemm.sh
+++ b/scripts/generate-qs8-gemm.sh
@@ -146,141 +146,231 @@
################################### x86 SSE ###################################
### C2 micro-kernels
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
-tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
+
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
+tools/xngen src/qs8-gemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
### C8 micro-kernels
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
-tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
+
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
+tools/xngen src/qs8-gemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=EXTENDED -o src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
################################### x86 AVX2 ##################################
### C8 micro-kernels
diff --git a/scripts/generate-qs8-igemm.sh b/scripts/generate-qs8-igemm.sh
index 8a21621..b888bb7 100755
--- a/scripts/generate-qs8-igemm.sh
+++ b/scripts/generate-qs8-igemm.sh
@@ -138,96 +138,186 @@
################################### x86 SSE ###################################
### C2 micro-kernels
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c2-sse.c.in -D MR=4 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
### C8 micro-kernels
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
-tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD64 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=2 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=3 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=0 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=0 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=GEMMLOWP -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=1 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=2 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
+tools/xngen src/qs8-igemm/MRx4c8-sse.c.in -D MR=3 -D SSE=4 -D AVX=1 -D XOP=1 -D REQUANTIZATION=FP32 -D VARIANT=LD128 -o src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
################################### x86 AVX2 ##################################
### C8 micro-kernels
diff --git a/src/init.c b/src/init.c
index 49ed439..fb353b3 100644
--- a/src/init.c
+++ b/src/init.c
@@ -1570,11 +1570,11 @@
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_xop()) {
// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1588,38 +1588,38 @@
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_avx()) {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_sse4_1()) {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_ssse3()) {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse2_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
} else {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse2_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
diff --git a/src/qs8-gemm/MRx4c2-sse.c.in b/src/qs8-gemm/MRx4c2-sse.c.in
index cdd7e71..6891ecf 100644
--- a/src/qs8-gemm/MRx4c2-sse.c.in
+++ b/src/qs8-gemm/MRx4c2-sse.c.in
@@ -6,6 +6,7 @@
$assert SSE in [2, 3, 4]
$assert not XOP or AVX
$assert not AVX or SSE == 4
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD64", "LD128", "EXTENDED"]
$assert MR <= 4
#include <assert.h>
@@ -27,9 +28,9 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
+$PARAMS_STRUCT = REQUANTIZATION.lower() + ("_sse4" if SSE >= 4 else "_sse2")
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
-void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x4c2__${ISA}${LOAD_SUFFIX}(
+void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x4c2__${ISA}${LOAD_SUFFIX}(
size_t mr,
size_t nc,
size_t kc,
@@ -215,90 +216,101 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE == 4:
+ $if SSE == 4:
+ $for M in range(MR):
+ const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
+ const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
+ $else:
+ $for M in range(MR):
+ const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
+
+ $for M in range(MR):
+ $if SSE >= 3:
+ const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
+ $else:
+ const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
+
+ $for M in range(MR):
+ const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
+ _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
- const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m128i vrem${M}x0123 =
+ _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
+
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $for M in range(MR):
+ vacc${M}x0123 =
+ _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m128 vscaled${M}x0123 = _mm_cvtepi32_ps(vacc${M}x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_${"sse4" if SSE >= 4 else "sse2"}.scale);
+ $for M in range(MR):
+ vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale);
$for M in range(MR):
- const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
- const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
- $else:
- $for M in range(MR):
- const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
-
- $for M in range(MR):
- $if SSE >= 3:
- const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
- $else:
- const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
-
- $for M in range(MR):
- const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
-
- $for M in range(MR):
- const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
- _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
-
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
- $for M in range(MR):
- const __m128i vrem${M}x0123 =
- _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
-
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $for M in range(MR):
- vacc${M}x0123 =
- _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ vacc${M}x0123 = _mm_cvtps_epi32(vscaled${M}x0123);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -308,8 +320,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/MRx4c8-sse.c.in b/src/qs8-gemm/MRx4c8-sse.c.in
index 6f26c05..7c505d9 100644
--- a/src/qs8-gemm/MRx4c8-sse.c.in
+++ b/src/qs8-gemm/MRx4c8-sse.c.in
@@ -6,6 +6,7 @@
$assert SSE in [2, 3, 4]
$assert not XOP or AVX
$assert not AVX or SSE == 4
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD64", "LD128", "EXTENDED"]
$assert MR <= 4
#include <assert.h>
@@ -27,9 +28,9 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
+$PARAMS_STRUCT = REQUANTIZATION.lower() + ("_sse4" if SSE >= 4 else "_sse2")
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
-void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x4c8__${ISA}${LOAD_SUFFIX}(
+void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x4c8__${ISA}${LOAD_SUFFIX}(
size_t mr,
size_t nc,
size_t kc,
@@ -152,90 +153,101 @@
$for M in range(MR):
__m128i vacc${M}x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc${M}x02, vacc${M}x13), _mm_unpackhi_epi32(vacc${M}x02, vacc${M}x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE == 4:
+ $if SSE == 4:
+ $for M in range(MR):
+ const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
+ const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
+ $else:
+ $for M in range(MR):
+ const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
+
+ $for M in range(MR):
+ $if SSE >= 3:
+ const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
+ $else:
+ const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
+
+ $for M in range(MR):
+ const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
+ _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
- const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m128i vrem${M}x0123 =
+ _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
+
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $for M in range(MR):
+ vacc${M}x0123 =
+ _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m128 vscaled${M}x0123 = _mm_cvtepi32_ps(vacc${M}x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_${"sse4" if SSE >= 4 else "sse2"}.scale);
+ $for M in range(MR):
+ vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale);
$for M in range(MR):
- const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
- const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
- $else:
- $for M in range(MR):
- const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
-
- $for M in range(MR):
- $if SSE >= 3:
- const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
- $else:
- const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
-
- $for M in range(MR):
- const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
-
- $for M in range(MR):
- const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
- _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
-
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
- $for M in range(MR):
- const __m128i vrem${M}x0123 =
- _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
-
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $for M in range(MR):
- vacc${M}x0123 =
- _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ vacc${M}x0123 = _mm_cvtps_epi32(vscaled${M}x0123);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -245,8 +257,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..841fdc5
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,145 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..ecc7bf0
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,145 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..202d416
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..3e8d08d
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..5ee38c5
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,145 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..4a61f66
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,145 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..145b151
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..6b207e9
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..c971ef9
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,150 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..cac9d9e
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,150 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..ed48fc0
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,117 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..da63466
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,119 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..da10e26
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,118 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..4b1bc7f
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,120 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..4009af4
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,117 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..7a806d0
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,119 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..ad24d67
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,118 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..641d3dd
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,120 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..5c20f88
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,122 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..9860e7b
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,124 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..c03b700
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..28c0f03
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..31f042a
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..cc1da18
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..dcbe48c
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..5b42ad3
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..055c4b3
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..90c0c93
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..bb8681d
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,186 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..a7b8771
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,186 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..791cbd4
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..ac2c116
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,148 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..3326d1e
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,148 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..5dc8d44
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,150 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..69ad9c2
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..3d02fa6
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,148 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..bff063f
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,148 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..43a1e38
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,150 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..eeeb805
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,151 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..53259fa
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,153 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..7bc2350
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,218 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..8d3eaa3
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,218 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..bf5d673
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,222 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..2040f86
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,222 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..77be744
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,218 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..8ccc871
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,218 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..26c0fd6
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,222 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..ef4a873
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,222 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..8d862f3
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,223 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..60cce10
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,223 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..167c725
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,176 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..1a54b46
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,178 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..d0c3877
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,180 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+ const __m128i vacc2x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x0, vacc2x2), _mm_unpackhi_epi32(vacc2x0, vacc2x2));
+ const __m128i vacc2x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x1, vacc2x3), _mm_unpackhi_epi32(vacc2x1, vacc2x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+ __m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..3bc4181
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,182 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+ const __m128i vacc2x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x0, vacc2x2), _mm_unpackhi_epi32(vacc2x0, vacc2x2));
+ const __m128i vacc2x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x1, vacc2x3), _mm_unpackhi_epi32(vacc2x1, vacc2x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+ __m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..dd4a934
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,176 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..6fe9768
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,178 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..0c042d1
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,180 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..6ae7371
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,182 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..4772e48
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
+ vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+ vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
+ vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..9dba8f5
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+ vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..cd5dce3
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,254 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..7d8c3e2
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,254 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..29dee36
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..2bb7875
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..539b395
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,254 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..edc063a
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,254 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..11f2641
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..ac3f801
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..9a82755
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..3ae703b
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/MRx4c2-sse.c.in b/src/qs8-igemm/MRx4c2-sse.c.in
index d08425d..9603e3a 100644
--- a/src/qs8-igemm/MRx4c2-sse.c.in
+++ b/src/qs8-igemm/MRx4c2-sse.c.in
@@ -6,6 +6,7 @@
$assert SSE in [2, 3, 4]
$assert not XOP or AVX
$assert not AVX or SSE == 4
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD64", "LD128"]
$assert MR <= 4
#include <assert.h>
@@ -25,9 +26,9 @@
#include <xnnpack/math.h>
-$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
+$PARAMS_STRUCT = REQUANTIZATION.lower() + ("_sse4" if SSE >= 4 else "_sse2")
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
-void xnn_qs8_igemm_minmax_gemmlowp_ukernel_${MR}x4c2__${ISA}_${VARIANT.lower()}(
+void xnn_qs8_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x4c2__${ISA}_${VARIANT.lower()}(
size_t mr,
size_t nc,
size_t kc,
@@ -193,82 +194,93 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE == 4:
+ $if SSE == 4:
+ $for M in range(MR):
+ const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
+ const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
+ $else:
+ $for M in range(MR):
+ const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
+
+ $for M in range(MR):
+ $if SSE >= 3:
+ const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
+ $else:
+ const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
+
+ $for M in range(MR):
+ const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
+ _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
- const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m128i vrem${M}x0123 =
+ _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
+
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $for M in range(MR):
+ vacc${M}x0123 =
+ _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m128 vscaled${M}x0123 = _mm_cvtepi32_ps(vacc${M}x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_${"sse4" if SSE >= 4 else "sse2"}.scale);
+ $for M in range(MR):
+ vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale);
$for M in range(MR):
- const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
- const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
- $else:
- $for M in range(MR):
- const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
-
- $for M in range(MR):
- $if SSE >= 3:
- const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
- $else:
- const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
-
- $for M in range(MR):
- const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
-
- $for M in range(MR):
- const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
- _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
-
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
- $for M in range(MR):
- const __m128i vrem${M}x0123 =
- _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
-
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $for M in range(MR):
- vacc${M}x0123 =
- _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ vacc${M}x0123 = _mm_cvtps_epi32(vscaled${M}x0123);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
diff --git a/src/qs8-igemm/MRx4c8-sse.c.in b/src/qs8-igemm/MRx4c8-sse.c.in
index b616122..86ce128 100644
--- a/src/qs8-igemm/MRx4c8-sse.c.in
+++ b/src/qs8-igemm/MRx4c8-sse.c.in
@@ -6,6 +6,7 @@
$assert SSE in [2, 3, 4]
$assert not XOP or AVX
$assert not AVX or SSE == 4
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD64", "LD128"]
$assert MR <= 4
#include <assert.h>
@@ -25,9 +26,9 @@
#include <xnnpack/math.h>
-$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
+$PARAMS_STRUCT = REQUANTIZATION.lower() + ("_sse4" if SSE >= 4 else "_sse2")
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
-void xnn_qs8_igemm_minmax_gemmlowp_ukernel_${MR}x4c8__${ISA}_${VARIANT.lower()}(
+void xnn_qs8_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x4c8__${ISA}_${VARIANT.lower()}(
size_t mr,
size_t nc,
size_t kc,
@@ -151,90 +152,101 @@
$for M in range(MR):
__m128i vacc${M}x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc${M}x02, vacc${M}x13), _mm_unpackhi_epi32(vacc${M}x02, vacc${M}x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE == 4:
+ $if SSE == 4:
+ $for M in range(MR):
+ const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
+ const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
+ $else:
+ $for M in range(MR):
+ const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
+
+ $for M in range(MR):
+ $if SSE >= 3:
+ const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
+ $else:
+ const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
+
+ $for M in range(MR):
+ const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
+ _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
- const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m128i vrem${M}x0123 =
+ _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
+
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $for M in range(MR):
+ vacc${M}x0123 =
+ _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m128 vscaled${M}x0123 = _mm_cvtepi32_ps(vacc${M}x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_${"sse4" if SSE >= 4 else "sse2"}.scale);
+ $for M in range(MR):
+ vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale);
$for M in range(MR):
- const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
- const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
- $else:
- $for M in range(MR):
- const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
-
- $for M in range(MR):
- $if SSE >= 3:
- const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
- $else:
- const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
-
- $for M in range(MR):
- const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
-
- $for M in range(MR):
- const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
- _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
-
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
- $for M in range(MR):
- const __m128i vrem${M}x0123 =
- _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
-
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $for M in range(MR):
- vacc${M}x0123 =
- _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ vacc${M}x0123 = _mm_cvtps_epi32(vscaled${M}x0123);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -244,8 +256,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max));
if (nc >= 4) {
$for M in reversed(range(1, MR)):
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..1d8f14c
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,157 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..66ea6ea
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,157 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..ac4c742
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,158 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..6c892af
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,158 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..866740d
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,157 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..091c8f0
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,157 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..daf3c09
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,158 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..ffc8d56
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,158 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..eaa6e92
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,162 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..1cb8c87
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,162 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..a81f6a4
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,129 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..ead06e3
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,131 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..4360fb7
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,130 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..b1cffaf
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,132 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..ef57058
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,129 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..dfc49bf
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,131 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..cc9b9c2
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,130 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..c19d705
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,132 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..406550e
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,134 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..2894376
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,136 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..ddc1725
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..a63c500
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..51cd3f6
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..6f16a59
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..1f99b51
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..3f96f5d
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..e4d15f9
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..f4d485b
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..00a4b2c
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,199 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..3710f0f
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,199 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..00d798f
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,159 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..56827ed
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,161 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..834cbdf
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,160 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..8f98520
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,162 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..e42f500
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,159 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..de772cb
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,161 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..d3f20da
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,160 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..88a422b
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,162 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..5c7af79
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,164 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..a725181
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,166 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..cde0021
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,232 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..cb62d7a
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,232 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..6385818
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,234 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..e5d00ed
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,234 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..02ef5f2
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,232 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..1ba142c
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,232 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..337b395
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,234 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..132a23e
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,234 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..0204f3a
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,237 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..2b600be
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,237 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..4116936
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,190 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..aa33476
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..bd5324f
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+ const __m128i vacc2x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x0, vacc2x2), _mm_unpackhi_epi32(vacc2x0, vacc2x2));
+ const __m128i vacc2x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x1, vacc2x3), _mm_unpackhi_epi32(vacc2x1, vacc2x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+ __m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..2a81f95
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+ const __m128i vacc2x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x0, vacc2x2), _mm_unpackhi_epi32(vacc2x0, vacc2x2));
+ const __m128i vacc2x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x1, vacc2x3), _mm_unpackhi_epi32(vacc2x1, vacc2x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+ __m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..6531f50
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,190 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..500f75d
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..8831628
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..036b585
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..883264f
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
+ vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+ vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
+ vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..439051b
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,197 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+ vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..10c39c7
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,269 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..c1fd952
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,269 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..31299ee
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,271 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..ea52094
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,271 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..1ba4d7d
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,269 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..f383fa7
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,269 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..9587ef0
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,271 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..363b742
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,271 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..08bc5cd
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,274 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..f660147
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,274 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/xnnpack/gemm.h b/src/xnnpack/gemm.h
index 445c098..b57e3ca 100644
--- a/src/xnnpack/gemm.h
+++ b/src/xnnpack/gemm.h
@@ -637,91 +637,181 @@
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c8__avx2)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x8c8__avx2)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x8c8__avx2)
diff --git a/src/xnnpack/igemm.h b/src/xnnpack/igemm.h
index ade91b9..eba12e4 100644
--- a/src/xnnpack/igemm.h
+++ b/src/xnnpack/igemm.h
@@ -440,91 +440,181 @@
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c8__avx2)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c8__avx2)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x8c8__avx2)
diff --git a/src/xnnpack/params-init.h b/src/xnnpack/params-init.h
index 2d81c53..885fa69 100644
--- a/src/xnnpack/params-init.h
+++ b/src/xnnpack/params-init.h
@@ -354,6 +354,42 @@
}
}
+static inline void xnn_init_qs8_conv_minmax_fp32_sse2_params(
+ union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ int8_t output_zero_point,
+ int8_t output_min,
+ int8_t output_max)
+{
+ for (uint32_t i = 0; i < 4; i++) {
+ params->fp32_sse2.scale[i] = scale;
+ }
+ for (uint32_t i = 0; i < 8; i++) {
+ params->fp32_sse2.output_zero_point[i] = (int16_t) output_zero_point;
+ params->fp32_sse2.output_min[i] = (int16_t) output_min;
+ params->fp32_sse2.output_max[i] = (int16_t) output_max;
+ }
+}
+
+static inline void xnn_init_qs8_conv_minmax_fp32_sse4_params(
+ union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ int8_t output_zero_point,
+ int8_t output_min,
+ int8_t output_max)
+{
+ for (uint32_t i = 0; i < 4; i++) {
+ params->fp32_sse4.scale[i] = scale;
+ }
+ for (uint32_t i = 0; i < 8; i++) {
+ params->fp32_sse4.output_zero_point[i] = (int16_t) output_zero_point;
+ }
+ for (uint32_t i = 0; i < 16; i++) {
+ params->fp32_sse4.output_min[i] = output_min;
+ params->fp32_sse4.output_max[i] = output_max;
+ }
+}
+
static inline void xnn_init_qs8_conv_minmax_fp32_avx2_params(
union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
float scale,
diff --git a/test/qs8-gemm-minmax-fp32.cc b/test/qs8-gemm-minmax-fp32.cc
index 3a51bd9..a923d7e 100644
--- a/test/qs8-gemm-minmax-fp32.cc
+++ b/test/qs8-gemm-minmax-fp32.cc
@@ -23,6 +23,31926 @@
#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
TEST(QS8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8) {
TEST_REQUIRES_X86_AVX2;
GemmMicrokernelTester()
diff --git a/test/qs8-gemm-minmax-fp32.yaml b/test/qs8-gemm-minmax-fp32.yaml
index 97ac4f7..f15fe7f 100644
--- a/test/qs8-gemm-minmax-fp32.yaml
+++ b/test/qs8-gemm-minmax-fp32.yaml
@@ -3,6 +3,216 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x8c8__avx2
init: xnn_init_qs8_conv_minmax_fp32_avx2_params
k-block: 8
diff --git a/test/qs8-igemm-minmax-fp32.cc b/test/qs8-igemm-minmax-fp32.cc
index a03bac3..7960a64 100644
--- a/test/qs8-igemm-minmax-fp32.cc
+++ b/test/qs8-igemm-minmax-fp32.cc
@@ -23,6 +23,31926 @@
#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C2__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C2__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C2__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X4C2__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(4)
+ .kr(2)
+ .sr(1)
+ .m(4)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD64, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD64, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD64, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE2_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSSE3;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, qmin) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, qmax) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSSE3_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSSE3;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128, xnn_init_qs8_conv_minmax_fp32_sse2_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_div_8) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_div_4) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_SSE41;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, qmin) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, qmax) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__SSE41_LD128, strided_cm) {
+ TEST_REQUIRES_X86_SSE41;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_div_8) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_div_4) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, qmin) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, qmax) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__AVX_LD128, strided_cm) {
+ TEST_REQUIRES_X86_AVX;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_1X4C8__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_2X4C8__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(4)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_lt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_gt_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_div_8) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, k_div_8_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_gt_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_gt_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_gt_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_gt_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 5; n < 8; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_div_4) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_div_4_strided_cn) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_div_4_strided_a) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, n_div_4_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (uint32_t n = 8; n <= 12; n += 4) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, strided_cm_subtile) {
+ TEST_REQUIRES_X86_XOP;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 4; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(7)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, qmin) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, qmax) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_3X4C8__XOP_LD128, strided_cm) {
+ TEST_REQUIRES_X86_XOP;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(4)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(4)
+ .k(8)
+ .cm_stride(7)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128, xnn_init_qs8_conv_minmax_fp32_sse4_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
TEST(QS8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8) {
TEST_REQUIRES_X86_AVX2;
GemmMicrokernelTester()
diff --git a/test/qs8-igemm-minmax-fp32.yaml b/test/qs8-igemm-minmax-fp32.yaml
index 280ad4f..2c64ba6 100644
--- a/test/qs8-igemm-minmax-fp32.yaml
+++ b/test/qs8-igemm-minmax-fp32.yaml
@@ -3,6 +3,216 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse2_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128
+ init: xnn_init_qs8_conv_minmax_fp32_sse4_params
+ k-block: 8
- name: xnn_qs8_igemm_minmax_fp32_ukernel_1x8c8__avx2
init: xnn_init_qs8_conv_minmax_fp32_avx2_params
k-block: 8