iOS 6x8 microkernel based on Cortex-A75 but with X18 avoided.
Use x14 for C pointer and reload cn_stride during clamping into x0.
PiperOrigin-RevId: 302555155
diff --git a/test/f32-gemm.cc b/test/f32-gemm.cc
index ec9b4b4..e56931e 100644
--- a/test/f32-gemm.cc
+++ b/test/f32-gemm.cc
@@ -7037,6 +7037,507 @@
#if XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(16)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(16)
+ .a_stride(19)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_16_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(16)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_lt_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_lt_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_lt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k < 16; k++) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_gt_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 17; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 17; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 17; k < 16; k++) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_div_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 24; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 24; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 24; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, n_gt_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, n_gt_8_strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, n_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, n_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, n_div_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, n_div_8_strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, n_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, n_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, qmin) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, qmax) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMM_6X8__AARCH64_NEONFMA_IOS, strided_cm) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+#endif // XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
+
+
+#if XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
TEST(F32_GEMM_1X12__AARCH64_NEONFMA_CORTEX_A53, k_eq_4) {
TEST_REQUIRES_ARM_NEON_FMA;
GemmMicrokernelTester()
diff --git a/test/f32-gemm.yaml b/test/f32-gemm.yaml
index 4ad8646..2408331 100644
--- a/test/f32-gemm.yaml
+++ b/test/f32-gemm.yaml
@@ -58,6 +58,10 @@
k-block: 8
pipelined: true
assembly: true
+- name: xnn_f32_gemm_ukernel_6x8__aarch64_neonfma_ios
+ k-block: 8
+ pipelined: true
+ assembly: true
- name: xnn_f32_gemm_ukernel_1x12__aarch64_neonfma_cortex_a53
k-block: 4
pipelined: true
diff --git a/test/f32-gemminc.cc b/test/f32-gemminc.cc
index ead6477..0a6da77 100644
--- a/test/f32-gemminc.cc
+++ b/test/f32-gemminc.cc
@@ -6491,6 +6491,507 @@
#if XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_eq_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_eq_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(16)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_eq_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(16)
+ .a_stride(19)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_eq_16_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(16)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_lt_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_lt_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_lt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k < 16; k++) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_gt_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 17; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 17; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 17; k < 16; k++) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_div_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 24; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 24; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 24; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, n_gt_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, n_gt_8_strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, n_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, n_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, n_div_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, n_div_8_strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, n_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, n_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, qmin) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, qmax) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_GEMMINC_6X8__AARCH64_NEONFMA_IOS, strided_cm) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios);
+ }
+#endif // XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
+
+
+#if XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
TEST(F32_GEMMINC_1X12__AARCH64_NEONFMA_CORTEX_A53, k_eq_4) {
TEST_REQUIRES_ARM_NEON_FMA;
GemmMicrokernelTester()
diff --git a/test/f32-gemminc.yaml b/test/f32-gemminc.yaml
index cb4670a..08d7c53 100644
--- a/test/f32-gemminc.yaml
+++ b/test/f32-gemminc.yaml
@@ -53,6 +53,10 @@
k-block: 8
pipelined: true
assembly: true
+- name: xnn_f32_gemminc_ukernel_6x8__aarch64_neonfma_ios
+ k-block: 8
+ pipelined: true
+ assembly: true
- name: xnn_f32_gemminc_ukernel_1x12__aarch64_neonfma_cortex_a53
k-block: 4
pipelined: true
diff --git a/test/f32-igemm.cc b/test/f32-igemm.cc
index 64d2bb2..fa54ce7 100644
--- a/test/f32-igemm.cc
+++ b/test/f32-igemm.cc
@@ -9473,6 +9473,505 @@
#if XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(16)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_eq_16_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(16)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_lt_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_lt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k < 16; k++) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_gt_16) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 17; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 17; k < 16; k++) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_div_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 24; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 24; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, n_gt_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, n_gt_8_strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, n_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, n_div_8) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, n_div_8_strided_cn) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, n_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, small_kernel) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, small_kernel_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, n_gt_8_small_kernel) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, n_div_8_small_kernel) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 6; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, a_offset) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .a_offset(251)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, zero) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (uint32_t mz = 0; mz < 6; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .a_offset(251)
+ .zero_index(mz)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+ }
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, qmin) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, qmax) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+
+ TEST(F32_IGEMM_6X8__AARCH64_NEONFMA_IOS, strided_cm) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ GemmMicrokernelTester()
+ .mr(6)
+ .nr(8)
+ .kr(1)
+ .sr(1)
+ .m(6)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios);
+ }
+#endif // XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
+
+
+#if XNN_ARCH_ARM64 && XNN_ENABLE_ASSEMBLY
TEST(F32_IGEMM_1X12__AARCH64_NEONFMA_CORTEX_A53, k_eq_4) {
TEST_REQUIRES_ARM_NEON_FMA;
GemmMicrokernelTester()
diff --git a/test/f32-igemm.yaml b/test/f32-igemm.yaml
index 5d742c3..4801bd2 100644
--- a/test/f32-igemm.yaml
+++ b/test/f32-igemm.yaml
@@ -73,6 +73,10 @@
k-block: 8
pipelined: true
assembly: true
+- name: xnn_f32_igemm_ukernel_6x8__aarch64_neonfma_ios
+ k-block: 8
+ pipelined: true
+ assembly: true
- name: xnn_f32_igemm_ukernel_1x12__aarch64_neonfma_cortex_a53
k-block: 4
pipelined: true