Support FP32 requantization in AVX2 QS8 microkernels
PiperOrigin-RevId: 375822588
diff --git a/src/init.c b/src/init.c
index a422804..a95df77 100644
--- a/src/init.c
+++ b/src/init.c
@@ -114,7 +114,7 @@
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c4__neondot);
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x8c4__neondot);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c4__neondot);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 2;
@@ -123,18 +123,18 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c2__neon_mlal_padal_dup);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c2__neon_mlal_padal_dup);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c2__neon_mlal_padal_dup);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 1;
}
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x9__neon_mul16;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[0].primary_tile = 9;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x25__neon_mul16;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
xnn_params.qs8.dwconv[1].primary_tile = 25;
@@ -845,7 +845,7 @@
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c4__neondot);
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c4__aarch64_neondot_ld64);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x16c4__neondot);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 16;
xnn_params.qs8.gemm.log2_kr = 2;
@@ -854,7 +854,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c8__aarch64_neon_mlal_padal);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c8__aarch64_neon_mlal_padal);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c8__aarch64_neon_mlal_padal);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -865,7 +865,7 @@
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c4__neondot);
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c4__neondot);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x16c4__neondot);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 16;
xnn_params.qs8.gemm.log2_kr = 2;
@@ -874,7 +874,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c2__neon_mlal_padal_dup);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c2__neon_mlal_padal_dup);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c2__neon_mlal_padal_dup);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 1;
@@ -895,7 +895,7 @@
}
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c4__neondot);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x16c4__neondot);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 16;
xnn_params.qs8.gemm.log2_kr = 2;
@@ -907,7 +907,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16__neon_mlal_lane);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x16__neon_mlal_lane);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 16;
break;
@@ -919,7 +919,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c8__aarch64_neon_mlal_padal_prfm);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c8__aarch64_neon_mlal_padal_prfm);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c8__aarch64_neon_mlal_padal_prfm);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -930,7 +930,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c8__aarch64_neon_mlal_padal);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c8__aarch64_neon_mlal_padal);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c8__aarch64_neon_mlal_padal);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -981,7 +981,7 @@
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c4__neondot);
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c4__neondot);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x16c4__neondot);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 16;
xnn_params.qs8.gemm.log2_kr = 2;
@@ -990,7 +990,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c2__neon_mlal_padal_dup);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c2__neon_mlal_padal_dup);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c2__neon_mlal_padal_dup);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 1;
@@ -999,11 +999,11 @@
#endif // XNN_PLATFORM_IOS || XNN_PLATFORM_MAC
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x9__neon_mul16;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[0].primary_tile = 9;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x25__neon_mul16;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_neon_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_neon_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
xnn_params.qs8.dwconv[1].primary_tile = 25;
@@ -1564,7 +1564,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c8__avx512skx);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c8__avx512skx);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x16c8__avx512skx);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 16;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1574,16 +1574,16 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_avx2()) {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x8c8__avx2);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x8c8__avx2);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c8__avx2);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c8__avx2);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_avx2_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x8c8__avx2);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x8c8__avx2);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x8c8__avx2);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x8c8__avx2);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx2_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1592,7 +1592,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1601,7 +1601,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1610,7 +1610,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_sse2_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1619,7 +1619,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_sse2_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1627,53 +1627,53 @@
if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up32x9__avx512skx_mul32;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_sse2_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
xnn_params.qs8.dwconv[0].channel_tile = 32;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up32x25__avx512skx_mul32;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_sse2_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
xnn_params.qs8.dwconv[1].channel_tile = 32;
} else if (cpuinfo_has_x86_xop()) {
// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x9__xop_mul32;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.dwconv[0].channel_tile = 16;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x25__xop_mul32;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.dwconv[1].channel_tile = 16;
} else if (cpuinfo_has_x86_avx2()) {
- xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x9__avx2_mul32;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_avx2_params;
+ xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__avx2_mul32;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx2_params;
xnn_params.qs8.dwconv[0].channel_tile = 16;
- xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x25__avx2_mul32;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_avx2_params;
+ xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__avx2_mul32;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx2_params;
xnn_params.qs8.dwconv[1].channel_tile = 16;
} else if (cpuinfo_has_x86_avx()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x9__avx_mul32;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.dwconv[0].channel_tile = 16;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x25__avx_mul32;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.dwconv[1].channel_tile = 16;
} else if (cpuinfo_has_x86_sse4_1()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x9__sse41_mul16;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x25__sse41_mul16;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_sse4_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
} else if (cpuinfo_has_x86_ssse3()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x9__ssse3_mul16;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_sse2_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x25__ssse3_mul16;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_sse2_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
} else if (cpuinfo_has_x86_sse2()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x9__sse2_mul16;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_sse2_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x25__sse2_mul16;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_sse2_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
}
xnn_params.qs8.dwconv[0].primary_tile = 9;
@@ -2306,17 +2306,17 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__wasmsimd_ld64);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__wasmsimd_ld64);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__wasmsimd_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_wasmsimd_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_wasmsimd_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x9__wasmsimd_mul16;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_wasmsimd_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_wasmsimd_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[0].primary_tile = 9;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x25__wasmsimd_mul16;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_wasmsimd_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_wasmsimd_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
xnn_params.qs8.dwconv[1].primary_tile = 25;
@@ -2843,7 +2843,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x2__scalar);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x2__scalar);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x2__scalar);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_scalar_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 2;
} else {
@@ -2851,17 +2851,17 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4__scalar);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4__scalar);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4__scalar);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_scalar_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 4;
}
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up2x9__scalar;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_scalar_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
xnn_params.qs8.dwconv[0].channel_tile = 2;
xnn_params.qs8.dwconv[0].primary_tile = 9;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up2x25__scalar;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_scalar_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
xnn_params.qs8.dwconv[1].channel_tile = 2;
xnn_params.qs8.dwconv[1].primary_tile = 25;
@@ -3229,16 +3229,16 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4__scalar);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4__scalar);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4__scalar);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_scalar_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up2x9__scalar;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_scalar_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
xnn_params.qs8.dwconv[0].channel_tile = 2;
xnn_params.qs8.dwconv[0].primary_tile = 9;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up2x25__scalar;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_scalar_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
xnn_params.qs8.dwconv[1].channel_tile = 2;
xnn_params.qs8.dwconv[1].primary_tile = 25;
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx2-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx2-mul16.c
new file mode 100644
index 0000000..e8938a0
--- /dev/null
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx2-mul16.c
@@ -0,0 +1,667 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul16.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__avx2_mul16(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+
+
+ const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
+ const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ i0 += 16;
+
+ const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
+ const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
+
+ const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
+ const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ i1 += 16;
+
+ const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
+ const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
+
+ const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
+ const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ i2 += 16;
+
+ const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
+ const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
+
+ const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
+ const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ i3 += 16;
+
+ const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
+ const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
+
+ const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
+ const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ i4 += 16;
+
+ const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
+ const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
+
+ const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
+ const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ i5 += 16;
+
+ const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
+ const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
+
+ const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
+ const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ i6 += 16;
+
+ const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
+ const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
+
+ const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
+ const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ i7 += 16;
+
+ const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
+ const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
+
+ const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
+ const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ i8 += 16;
+
+ const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
+ const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
+
+ const __m256i vi9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i9));
+ const __m256i vk9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ i9 += 16;
+
+ const __m256i vprod9x0123456789ABCDEF = _mm256_mullo_epi16(vi9x0123456789ABCDEF, vk9x0123456789ABCDEF);
+ const __m128i vprod9x89ABCDEF = _mm256_extracti128_si256(vprod9x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod9x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod9x89ABCDEF));
+
+ const __m256i vi10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i10));
+ const __m256i vk10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ i10 += 16;
+
+ const __m256i vprod10x0123456789ABCDEF = _mm256_mullo_epi16(vi10x0123456789ABCDEF, vk10x0123456789ABCDEF);
+ const __m128i vprod10x89ABCDEF = _mm256_extracti128_si256(vprod10x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod10x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod10x89ABCDEF));
+
+ const __m256i vi11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i11));
+ const __m256i vk11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ i11 += 16;
+
+ const __m256i vprod11x0123456789ABCDEF = _mm256_mullo_epi16(vi11x0123456789ABCDEF, vk11x0123456789ABCDEF);
+ const __m128i vprod11x89ABCDEF = _mm256_extracti128_si256(vprod11x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod11x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod11x89ABCDEF));
+
+ const __m256i vi12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i12));
+ const __m256i vk12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ i12 += 16;
+
+ const __m256i vprod12x0123456789ABCDEF = _mm256_mullo_epi16(vi12x0123456789ABCDEF, vk12x0123456789ABCDEF);
+ const __m128i vprod12x89ABCDEF = _mm256_extracti128_si256(vprod12x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod12x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod12x89ABCDEF));
+
+ const __m256i vi13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i13));
+ const __m256i vk13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+ i13 += 16;
+
+ const __m256i vprod13x0123456789ABCDEF = _mm256_mullo_epi16(vi13x0123456789ABCDEF, vk13x0123456789ABCDEF);
+ const __m128i vprod13x89ABCDEF = _mm256_extracti128_si256(vprod13x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod13x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod13x89ABCDEF));
+
+ const __m256i vi14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i14));
+ const __m256i vk14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 224 * sizeof(int8_t))));
+ i14 += 16;
+
+ const __m256i vprod14x0123456789ABCDEF = _mm256_mullo_epi16(vi14x0123456789ABCDEF, vk14x0123456789ABCDEF);
+ const __m128i vprod14x89ABCDEF = _mm256_extracti128_si256(vprod14x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod14x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod14x89ABCDEF));
+
+ const __m256i vi15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i15));
+ const __m256i vk15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 240 * sizeof(int8_t))));
+ i15 += 16;
+
+ const __m256i vprod15x0123456789ABCDEF = _mm256_mullo_epi16(vi15x0123456789ABCDEF, vk15x0123456789ABCDEF);
+ const __m128i vprod15x89ABCDEF = _mm256_extracti128_si256(vprod15x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod15x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod15x89ABCDEF));
+
+ const __m256i vi16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i16));
+ const __m256i vk16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 256 * sizeof(int8_t))));
+ i16 += 16;
+
+ const __m256i vprod16x0123456789ABCDEF = _mm256_mullo_epi16(vi16x0123456789ABCDEF, vk16x0123456789ABCDEF);
+ const __m128i vprod16x89ABCDEF = _mm256_extracti128_si256(vprod16x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod16x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod16x89ABCDEF));
+
+ const __m256i vi17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i17));
+ const __m256i vk17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 272 * sizeof(int8_t))));
+ i17 += 16;
+
+ const __m256i vprod17x0123456789ABCDEF = _mm256_mullo_epi16(vi17x0123456789ABCDEF, vk17x0123456789ABCDEF);
+ const __m128i vprod17x89ABCDEF = _mm256_extracti128_si256(vprod17x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod17x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod17x89ABCDEF));
+
+ const __m256i vi18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i18));
+ const __m256i vk18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 288 * sizeof(int8_t))));
+ i18 += 16;
+
+ const __m256i vprod18x0123456789ABCDEF = _mm256_mullo_epi16(vi18x0123456789ABCDEF, vk18x0123456789ABCDEF);
+ const __m128i vprod18x89ABCDEF = _mm256_extracti128_si256(vprod18x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod18x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod18x89ABCDEF));
+
+ const __m256i vi19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i19));
+ const __m256i vk19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 304 * sizeof(int8_t))));
+ i19 += 16;
+
+ const __m256i vprod19x0123456789ABCDEF = _mm256_mullo_epi16(vi19x0123456789ABCDEF, vk19x0123456789ABCDEF);
+ const __m128i vprod19x89ABCDEF = _mm256_extracti128_si256(vprod19x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod19x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod19x89ABCDEF));
+
+ const __m256i vi20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i20));
+ const __m256i vk20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 320 * sizeof(int8_t))));
+ i20 += 16;
+
+ const __m256i vprod20x0123456789ABCDEF = _mm256_mullo_epi16(vi20x0123456789ABCDEF, vk20x0123456789ABCDEF);
+ const __m128i vprod20x89ABCDEF = _mm256_extracti128_si256(vprod20x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod20x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod20x89ABCDEF));
+
+ const __m256i vi21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i21));
+ const __m256i vk21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 336 * sizeof(int8_t))));
+ i21 += 16;
+
+ const __m256i vprod21x0123456789ABCDEF = _mm256_mullo_epi16(vi21x0123456789ABCDEF, vk21x0123456789ABCDEF);
+ const __m128i vprod21x89ABCDEF = _mm256_extracti128_si256(vprod21x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod21x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod21x89ABCDEF));
+
+ const __m256i vi22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i22));
+ const __m256i vk22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 352 * sizeof(int8_t))));
+ i22 += 16;
+
+ const __m256i vprod22x0123456789ABCDEF = _mm256_mullo_epi16(vi22x0123456789ABCDEF, vk22x0123456789ABCDEF);
+ const __m128i vprod22x89ABCDEF = _mm256_extracti128_si256(vprod22x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod22x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod22x89ABCDEF));
+
+ const __m256i vi23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i23));
+ const __m256i vk23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 368 * sizeof(int8_t))));
+ i23 += 16;
+
+ const __m256i vprod23x0123456789ABCDEF = _mm256_mullo_epi16(vi23x0123456789ABCDEF, vk23x0123456789ABCDEF);
+ const __m128i vprod23x89ABCDEF = _mm256_extracti128_si256(vprod23x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod23x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod23x89ABCDEF));
+
+ const __m256i vi24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i24));
+ const __m256i vk24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 384 * sizeof(int8_t))));
+ i24 += 16;
+
+ const __m256i vprod24x0123456789ABCDEF = _mm256_mullo_epi16(vi24x0123456789ABCDEF, vk24x0123456789ABCDEF);
+ const __m128i vprod24x89ABCDEF = _mm256_extracti128_si256(vprod24x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod24x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
+
+ w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+
+
+ const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
+ const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+
+ const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
+ const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
+
+ const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
+ const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+
+ const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
+ const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
+
+ const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
+ const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+
+ const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
+ const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
+
+ const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
+ const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+
+ const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
+ const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
+
+ const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
+ const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+
+ const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
+ const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
+
+ const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
+ const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+
+ const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
+ const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
+
+ const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
+ const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+
+ const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
+ const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
+
+ const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
+ const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+
+ const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
+ const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
+
+ const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
+ const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+
+ const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
+ const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
+
+ const __m256i vi9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i9));
+ const __m256i vk9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+
+ const __m256i vprod9x0123456789ABCDEF = _mm256_mullo_epi16(vi9x0123456789ABCDEF, vk9x0123456789ABCDEF);
+ const __m128i vprod9x89ABCDEF = _mm256_extracti128_si256(vprod9x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod9x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod9x89ABCDEF));
+
+ const __m256i vi10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i10));
+ const __m256i vk10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+
+ const __m256i vprod10x0123456789ABCDEF = _mm256_mullo_epi16(vi10x0123456789ABCDEF, vk10x0123456789ABCDEF);
+ const __m128i vprod10x89ABCDEF = _mm256_extracti128_si256(vprod10x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod10x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod10x89ABCDEF));
+
+ const __m256i vi11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i11));
+ const __m256i vk11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+
+ const __m256i vprod11x0123456789ABCDEF = _mm256_mullo_epi16(vi11x0123456789ABCDEF, vk11x0123456789ABCDEF);
+ const __m128i vprod11x89ABCDEF = _mm256_extracti128_si256(vprod11x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod11x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod11x89ABCDEF));
+
+ const __m256i vi12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i12));
+ const __m256i vk12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+
+ const __m256i vprod12x0123456789ABCDEF = _mm256_mullo_epi16(vi12x0123456789ABCDEF, vk12x0123456789ABCDEF);
+ const __m128i vprod12x89ABCDEF = _mm256_extracti128_si256(vprod12x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod12x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod12x89ABCDEF));
+
+ const __m256i vi13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i13));
+ const __m256i vk13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+
+ const __m256i vprod13x0123456789ABCDEF = _mm256_mullo_epi16(vi13x0123456789ABCDEF, vk13x0123456789ABCDEF);
+ const __m128i vprod13x89ABCDEF = _mm256_extracti128_si256(vprod13x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod13x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod13x89ABCDEF));
+
+ const __m256i vi14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i14));
+ const __m256i vk14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 224 * sizeof(int8_t))));
+
+ const __m256i vprod14x0123456789ABCDEF = _mm256_mullo_epi16(vi14x0123456789ABCDEF, vk14x0123456789ABCDEF);
+ const __m128i vprod14x89ABCDEF = _mm256_extracti128_si256(vprod14x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod14x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod14x89ABCDEF));
+
+ const __m256i vi15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i15));
+ const __m256i vk15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 240 * sizeof(int8_t))));
+
+ const __m256i vprod15x0123456789ABCDEF = _mm256_mullo_epi16(vi15x0123456789ABCDEF, vk15x0123456789ABCDEF);
+ const __m128i vprod15x89ABCDEF = _mm256_extracti128_si256(vprod15x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod15x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod15x89ABCDEF));
+
+ const __m256i vi16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i16));
+ const __m256i vk16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 256 * sizeof(int8_t))));
+
+ const __m256i vprod16x0123456789ABCDEF = _mm256_mullo_epi16(vi16x0123456789ABCDEF, vk16x0123456789ABCDEF);
+ const __m128i vprod16x89ABCDEF = _mm256_extracti128_si256(vprod16x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod16x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod16x89ABCDEF));
+
+ const __m256i vi17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i17));
+ const __m256i vk17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 272 * sizeof(int8_t))));
+
+ const __m256i vprod17x0123456789ABCDEF = _mm256_mullo_epi16(vi17x0123456789ABCDEF, vk17x0123456789ABCDEF);
+ const __m128i vprod17x89ABCDEF = _mm256_extracti128_si256(vprod17x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod17x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod17x89ABCDEF));
+
+ const __m256i vi18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i18));
+ const __m256i vk18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 288 * sizeof(int8_t))));
+
+ const __m256i vprod18x0123456789ABCDEF = _mm256_mullo_epi16(vi18x0123456789ABCDEF, vk18x0123456789ABCDEF);
+ const __m128i vprod18x89ABCDEF = _mm256_extracti128_si256(vprod18x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod18x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod18x89ABCDEF));
+
+ const __m256i vi19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i19));
+ const __m256i vk19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 304 * sizeof(int8_t))));
+
+ const __m256i vprod19x0123456789ABCDEF = _mm256_mullo_epi16(vi19x0123456789ABCDEF, vk19x0123456789ABCDEF);
+ const __m128i vprod19x89ABCDEF = _mm256_extracti128_si256(vprod19x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod19x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod19x89ABCDEF));
+
+ const __m256i vi20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i20));
+ const __m256i vk20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 320 * sizeof(int8_t))));
+
+ const __m256i vprod20x0123456789ABCDEF = _mm256_mullo_epi16(vi20x0123456789ABCDEF, vk20x0123456789ABCDEF);
+ const __m128i vprod20x89ABCDEF = _mm256_extracti128_si256(vprod20x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod20x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod20x89ABCDEF));
+
+ const __m256i vi21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i21));
+ const __m256i vk21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 336 * sizeof(int8_t))));
+
+ const __m256i vprod21x0123456789ABCDEF = _mm256_mullo_epi16(vi21x0123456789ABCDEF, vk21x0123456789ABCDEF);
+ const __m128i vprod21x89ABCDEF = _mm256_extracti128_si256(vprod21x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod21x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod21x89ABCDEF));
+
+ const __m256i vi22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i22));
+ const __m256i vk22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 352 * sizeof(int8_t))));
+
+ const __m256i vprod22x0123456789ABCDEF = _mm256_mullo_epi16(vi22x0123456789ABCDEF, vk22x0123456789ABCDEF);
+ const __m128i vprod22x89ABCDEF = _mm256_extracti128_si256(vprod22x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod22x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod22x89ABCDEF));
+
+ const __m256i vi23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i23));
+ const __m256i vk23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 368 * sizeof(int8_t))));
+
+ const __m256i vprod23x0123456789ABCDEF = _mm256_mullo_epi16(vi23x0123456789ABCDEF, vk23x0123456789ABCDEF);
+ const __m128i vprod23x89ABCDEF = _mm256_extracti128_si256(vprod23x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod23x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod23x89ABCDEF));
+
+ const __m256i vi24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i24));
+ const __m256i vk24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 384 * sizeof(int8_t))));
+
+ const __m256i vprod24x0123456789ABCDEF = _mm256_mullo_epi16(vi24x0123456789ABCDEF, vk24x0123456789ABCDEF);
+ const __m128i vprod24x89ABCDEF = _mm256_extracti128_si256(vprod24x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod24x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
+
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+ __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+
+ __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
+
+ if (c & 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
+ vout0123456789ABCDEF = _mm_unpackhi_epi64(vout0123456789ABCDEF, vout0123456789ABCDEF);
+ output += 8;
+ }
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456789ABCDEF);
+ vout0123456789ABCDEF = _mm_srli_epi64(vout0123456789ABCDEF, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456789ABCDEF, 0);
+ vout0123456789ABCDEF = _mm_srli_epi32(vout0123456789ABCDEF, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456789ABCDEF, 0);
+ output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx2-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx2-mul32.c
new file mode 100644
index 0000000..4cf1156
--- /dev/null
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx2-mul32.c
@@ -0,0 +1,615 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul32.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__avx2_mul32(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
+ const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+ i0 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
+ const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+ i1 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
+ const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+ i2 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
+ const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+ i3 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
+ const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t))));
+ i4 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
+ const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t))));
+ i5 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
+ const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t))));
+ i6 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
+ const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t))));
+ i7 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ const __m256i vi8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 8)));
+ const __m256i vk8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t))));
+ i8 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi8x89ABCDEF, vk8x89ABCDEF));
+
+ const __m256i vi9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i9));
+ const __m256i vk9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ const __m256i vi9x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i9 + 8)));
+ const __m256i vk9x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 152 * sizeof(int8_t))));
+ i9 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi9x01234567, vk9x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi9x89ABCDEF, vk9x89ABCDEF));
+
+ const __m256i vi10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i10));
+ const __m256i vk10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ const __m256i vi10x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i10 + 8)));
+ const __m256i vk10x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 168 * sizeof(int8_t))));
+ i10 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi10x01234567, vk10x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi10x89ABCDEF, vk10x89ABCDEF));
+
+ const __m256i vi11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i11));
+ const __m256i vk11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ const __m256i vi11x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i11 + 8)));
+ const __m256i vk11x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 184 * sizeof(int8_t))));
+ i11 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi11x01234567, vk11x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi11x89ABCDEF, vk11x89ABCDEF));
+
+ const __m256i vi12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i12));
+ const __m256i vk12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ const __m256i vi12x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i12 + 8)));
+ const __m256i vk12x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 200 * sizeof(int8_t))));
+ i12 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi12x01234567, vk12x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi12x89ABCDEF, vk12x89ABCDEF));
+
+ const __m256i vi13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i13));
+ const __m256i vk13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+ const __m256i vi13x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i13 + 8)));
+ const __m256i vk13x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 216 * sizeof(int8_t))));
+ i13 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi13x01234567, vk13x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi13x89ABCDEF, vk13x89ABCDEF));
+
+ const __m256i vi14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i14));
+ const __m256i vk14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 224 * sizeof(int8_t))));
+ const __m256i vi14x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i14 + 8)));
+ const __m256i vk14x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 232 * sizeof(int8_t))));
+ i14 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi14x01234567, vk14x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi14x89ABCDEF, vk14x89ABCDEF));
+
+ const __m256i vi15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i15));
+ const __m256i vk15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 240 * sizeof(int8_t))));
+ const __m256i vi15x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i15 + 8)));
+ const __m256i vk15x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 248 * sizeof(int8_t))));
+ i15 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi15x01234567, vk15x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi15x89ABCDEF, vk15x89ABCDEF));
+
+ const __m256i vi16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i16));
+ const __m256i vk16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 256 * sizeof(int8_t))));
+ const __m256i vi16x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i16 + 8)));
+ const __m256i vk16x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 264 * sizeof(int8_t))));
+ i16 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi16x01234567, vk16x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi16x89ABCDEF, vk16x89ABCDEF));
+
+ const __m256i vi17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i17));
+ const __m256i vk17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 272 * sizeof(int8_t))));
+ const __m256i vi17x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i17 + 8)));
+ const __m256i vk17x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 280 * sizeof(int8_t))));
+ i17 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi17x01234567, vk17x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi17x89ABCDEF, vk17x89ABCDEF));
+
+ const __m256i vi18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i18));
+ const __m256i vk18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 288 * sizeof(int8_t))));
+ const __m256i vi18x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i18 + 8)));
+ const __m256i vk18x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 296 * sizeof(int8_t))));
+ i18 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi18x01234567, vk18x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi18x89ABCDEF, vk18x89ABCDEF));
+
+ const __m256i vi19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i19));
+ const __m256i vk19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 304 * sizeof(int8_t))));
+ const __m256i vi19x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i19 + 8)));
+ const __m256i vk19x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 312 * sizeof(int8_t))));
+ i19 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi19x01234567, vk19x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi19x89ABCDEF, vk19x89ABCDEF));
+
+ const __m256i vi20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i20));
+ const __m256i vk20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 320 * sizeof(int8_t))));
+ const __m256i vi20x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i20 + 8)));
+ const __m256i vk20x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 328 * sizeof(int8_t))));
+ i20 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi20x01234567, vk20x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi20x89ABCDEF, vk20x89ABCDEF));
+
+ const __m256i vi21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i21));
+ const __m256i vk21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 336 * sizeof(int8_t))));
+ const __m256i vi21x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i21 + 8)));
+ const __m256i vk21x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 344 * sizeof(int8_t))));
+ i21 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi21x01234567, vk21x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi21x89ABCDEF, vk21x89ABCDEF));
+
+ const __m256i vi22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i22));
+ const __m256i vk22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 352 * sizeof(int8_t))));
+ const __m256i vi22x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i22 + 8)));
+ const __m256i vk22x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 360 * sizeof(int8_t))));
+ i22 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi22x01234567, vk22x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi22x89ABCDEF, vk22x89ABCDEF));
+
+ const __m256i vi23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i23));
+ const __m256i vk23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 368 * sizeof(int8_t))));
+ const __m256i vi23x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i23 + 8)));
+ const __m256i vk23x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 376 * sizeof(int8_t))));
+ i23 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi23x01234567, vk23x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi23x89ABCDEF, vk23x89ABCDEF));
+
+ const __m256i vi24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i24));
+ const __m256i vk24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 384 * sizeof(int8_t))));
+ const __m256i vi24x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i24 + 8)));
+ const __m256i vk24x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 392 * sizeof(int8_t))));
+ i24 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi24x89ABCDEF, vk24x89ABCDEF));
+
+ w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ do {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) k));
+ i0 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 16)));
+ i1 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 32)));
+ i2 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 48)));
+ i3 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 64)));
+ i4 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 80)));
+ i5 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 96)));
+ i6 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 112)));
+ i7 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 128)));
+ i8 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ const __m256i vi9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i9));
+ const __m256i vk9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 144)));
+ i9 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi9x01234567, vk9x01234567));
+
+ const __m256i vi10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i10));
+ const __m256i vk10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 160)));
+ i10 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi10x01234567, vk10x01234567));
+
+ const __m256i vi11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i11));
+ const __m256i vk11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 176)));
+ i11 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi11x01234567, vk11x01234567));
+
+ const __m256i vi12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i12));
+ const __m256i vk12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 192)));
+ i12 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi12x01234567, vk12x01234567));
+
+ const __m256i vi13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i13));
+ const __m256i vk13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 208)));
+ i13 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi13x01234567, vk13x01234567));
+
+ const __m256i vi14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i14));
+ const __m256i vk14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 224)));
+ i14 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi14x01234567, vk14x01234567));
+
+ const __m256i vi15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i15));
+ const __m256i vk15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 240)));
+ i15 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi15x01234567, vk15x01234567));
+
+ const __m256i vi16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i16));
+ const __m256i vk16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 256)));
+ i16 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi16x01234567, vk16x01234567));
+
+ const __m256i vi17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i17));
+ const __m256i vk17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 272)));
+ i17 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi17x01234567, vk17x01234567));
+
+ const __m256i vi18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i18));
+ const __m256i vk18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 288)));
+ i18 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi18x01234567, vk18x01234567));
+
+ const __m256i vi19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i19));
+ const __m256i vk19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 304)));
+ i19 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi19x01234567, vk19x01234567));
+
+ const __m256i vi20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i20));
+ const __m256i vk20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 320)));
+ i20 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi20x01234567, vk20x01234567));
+
+ const __m256i vi21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i21));
+ const __m256i vk21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 336)));
+ i21 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi21x01234567, vk21x01234567));
+
+ const __m256i vi22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i22));
+ const __m256i vk22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 352)));
+ i22 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi22x01234567, vk22x01234567));
+
+ const __m256i vi23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i23));
+ const __m256i vk23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 368)));
+ i23 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi23x01234567, vk23x01234567));
+
+ const __m256i vi24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i24));
+ const __m256i vk24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 384)));
+ i24 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
+
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ k += 8;
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ if XNN_LIKELY(c >= 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456701234567);
+ output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
+ vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456701234567, 0);
+ vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
+ output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul16.c
index b7e5dee..0319710 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul16.c
@@ -693,8 +693,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -723,7 +723,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -733,8 +733,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -744,17 +744,17 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
@@ -1095,8 +1095,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -1115,27 +1115,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul32.c
index 7014a60..559b173 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul32.c
@@ -544,8 +544,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -574,7 +574,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -584,8 +584,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -595,12 +595,12 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -767,8 +767,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -780,21 +780,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx2-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx2-mul16.c
index b849733..ffd862a 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx2-mul16.c
@@ -390,9 +390,8 @@
vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
-
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -410,26 +409,26 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -643,8 +642,8 @@
vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -662,25 +661,25 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx2-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx2-mul32.c
index 810d524..9db5041 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx2-mul32.c
@@ -391,8 +391,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -410,26 +410,26 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -595,8 +595,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -608,21 +608,21 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx512skx-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx512skx-mul32.c
index ef0bc78..891b6ca 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx512skx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx512skx-mul32.c
@@ -31,14 +31,14 @@
assert(output_width != 0);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
do {
const int8_t* i0 = input[0];
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-neon-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-neon-mul16.c
index 2b6e60e..02e88f5 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-neon-mul16.c
@@ -29,12 +29,12 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse2-mul16.c
index a14750c..cf1ace0 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse2-mul16.c
@@ -693,8 +693,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -760,7 +760,7 @@
const __m128i vq31prod89AB = _mm_shuffle_epi32(vq31prod8A9B, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prodCDEF = _mm_shuffle_epi32(vq31prodCEDF, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -770,8 +770,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -781,15 +781,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
@@ -1134,8 +1134,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -1174,24 +1174,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul16.c
index cff6ab9..2de930e 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul16.c
@@ -693,8 +693,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -723,7 +723,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -733,8 +733,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -744,17 +744,17 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
@@ -1095,8 +1095,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -1115,27 +1115,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul32.c
index 1b5ca78..46063dc 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul32.c
@@ -544,8 +544,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -574,7 +574,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -584,8 +584,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -595,12 +595,12 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -767,8 +767,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -780,21 +780,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-ssse3-mul16.c
index 11496f8..74e7d01 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-ssse3-mul16.c
@@ -693,8 +693,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -760,7 +760,7 @@
const __m128i vq31prod89AB = _mm_shuffle_epi32(vq31prod8A9B, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prodCDEF = _mm_shuffle_epi32(vq31prodCEDF, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -770,8 +770,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -781,15 +781,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
@@ -1134,8 +1134,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -1174,24 +1174,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-wasmsimd-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-wasmsimd-mul16.c
index fc47ece..39c5ac8 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-wasmsimd-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-wasmsimd-mul16.c
@@ -532,8 +532,8 @@
const v128_t vaccCD = wasm_v32x4_shuffle(vaccCDEF, vsignCDEF, 0, 4, 1, 5);
const v128_t vaccEF = wasm_v32x4_shuffle(vaccCDEF, vsignCDEF, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -548,25 +548,25 @@
const v128_t vq31prod89AB = wasm_v32x4_shuffle(vprod89, vprodAB, 1, 3, 5, 7);
const v128_t vq31prodCDEF = wasm_v32x4_shuffle(vprodCD, vprodEF, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
const v128_t vrem89AB = wasm_i32x4_add(wasm_v128_and(vq31prod89AB, vremainder_mask), wasm_i32x4_shr(vq31prod89AB, 31));
const v128_t vremCDEF = wasm_i32x4_add(wasm_v128_and(vq31prodCDEF, vremainder_mask), wasm_i32x4_shr(vq31prodCDEF, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
vacc89AB = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod89AB, vshift), wasm_i32x4_gt(vrem89AB, vthreshold));
vaccCDEF = wasm_i32x4_sub(wasm_i32x4_shr(vq31prodCDEF, vshift), wasm_i32x4_gt(vremCDEF, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
v128_t vout89ABCDEF = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456789ABCDEF = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF), voutput_min), voutput_max);
wasm_v128_store(output, vout0123456789ABCDEF);
@@ -815,8 +815,8 @@
const v128_t vacc45 = wasm_v32x4_shuffle(vacc4567, vsign4567, 0, 4, 1, 5);
const v128_t vacc67 = wasm_v32x4_shuffle(vacc4567, vsign4567, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -825,20 +825,20 @@
const v128_t vq31prod0123 = wasm_v32x4_shuffle(vprod01, vprod23, 1, 3, 5, 7);
const v128_t vq31prod4567 = wasm_v32x4_shuffle(vprod45, vprod67, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
if XNN_LIKELY(c >= 8) {
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c
index c81791e..0f0e92d 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c
@@ -549,8 +549,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -579,7 +579,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -589,8 +589,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -600,12 +600,12 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -772,8 +772,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -785,21 +785,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx2-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx2-mul16.c
new file mode 100644
index 0000000..9a002b0
--- /dev/null
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx2-mul16.c
@@ -0,0 +1,315 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul16.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__avx2_mul16(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+
+
+ const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
+ const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ i0 += 16;
+
+ const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
+ const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
+
+ const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
+ const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ i1 += 16;
+
+ const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
+ const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
+
+ const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
+ const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ i2 += 16;
+
+ const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
+ const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
+
+ const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
+ const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ i3 += 16;
+
+ const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
+ const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
+
+ const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
+ const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ i4 += 16;
+
+ const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
+ const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
+
+ const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
+ const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ i5 += 16;
+
+ const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
+ const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
+
+ const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
+ const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ i6 += 16;
+
+ const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
+ const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
+
+ const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
+ const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ i7 += 16;
+
+ const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
+ const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
+
+ const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
+ const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ i8 += 16;
+
+ const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
+ const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
+
+ w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+
+
+ const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
+ const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+
+ const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
+ const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
+
+ const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
+ const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+
+ const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
+ const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
+
+ const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
+ const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+
+ const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
+ const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
+
+ const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
+ const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+
+ const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
+ const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
+
+ const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
+ const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+
+ const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
+ const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
+
+ const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
+ const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+
+ const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
+ const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
+
+ const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
+ const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+
+ const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
+ const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
+
+ const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
+ const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+
+ const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
+ const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
+
+ const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
+ const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+
+ const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
+ const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
+
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+ __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+
+ __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
+
+ if (c & 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
+ vout0123456789ABCDEF = _mm_unpackhi_epi64(vout0123456789ABCDEF, vout0123456789ABCDEF);
+ output += 8;
+ }
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456789ABCDEF);
+ vout0123456789ABCDEF = _mm_srli_epi64(vout0123456789ABCDEF, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456789ABCDEF, 0);
+ vout0123456789ABCDEF = _mm_srli_epi32(vout0123456789ABCDEF, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456789ABCDEF, 0);
+ output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx2-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx2-mul32.c
new file mode 100644
index 0000000..9b34f93
--- /dev/null
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx2-mul32.c
@@ -0,0 +1,295 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul32.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__avx2_mul32(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
+ const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+ i0 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
+ const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+ i1 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
+ const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+ i2 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
+ const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+ i3 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
+ const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t))));
+ i4 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
+ const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t))));
+ i5 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
+ const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t))));
+ i6 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
+ const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t))));
+ i7 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ const __m256i vi8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 8)));
+ const __m256i vk8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t))));
+ i8 += 16;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi8x89ABCDEF, vk8x89ABCDEF));
+
+ w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ do {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) k));
+ i0 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 16)));
+ i1 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 32)));
+ i2 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 48)));
+ i3 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 64)));
+ i4 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 80)));
+ i5 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 96)));
+ i6 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 112)));
+ i7 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 128)));
+ i8 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ k += 8;
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ if XNN_LIKELY(c >= 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456701234567);
+ output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
+ vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456701234567, 0);
+ vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
+ output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul16.c
index 70819c4..522eafb 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul16.c
@@ -277,8 +277,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -307,7 +307,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -317,8 +317,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -328,17 +328,17 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
@@ -471,8 +471,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -491,27 +491,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul32.c
index 5ba8e77..3132d70 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul32.c
@@ -224,8 +224,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -254,7 +254,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -264,8 +264,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -275,12 +275,12 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -351,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -364,21 +364,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx2-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx2-mul16.c
index 5587b9e..5476ae8 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx2-mul16.c
@@ -166,9 +166,8 @@
vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
-
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -186,26 +185,26 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -291,8 +290,8 @@
vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -310,25 +309,25 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx2-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx2-mul32.c
index 2b3af4e..a52e31a 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx2-mul32.c
@@ -167,8 +167,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -186,26 +186,26 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -275,8 +275,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -288,21 +288,21 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx512skx-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx512skx-mul32.c
index 2d92800..c6355b6 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx512skx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx512skx-mul32.c
@@ -31,14 +31,14 @@
assert(output_width != 0);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
do {
const int8_t* i0 = input[0];
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-neon-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-neon-mul16.c
index d21e9fc..d9159c1 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-neon-mul16.c
@@ -29,12 +29,12 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse2-mul16.c
index 17f7b55..de2556b 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse2-mul16.c
@@ -277,8 +277,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -344,7 +344,7 @@
const __m128i vq31prod89AB = _mm_shuffle_epi32(vq31prod8A9B, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prodCDEF = _mm_shuffle_epi32(vq31prodCEDF, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -354,8 +354,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -365,15 +365,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
@@ -510,8 +510,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -550,24 +550,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul16.c
index 6249771..3c6377c 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul16.c
@@ -277,8 +277,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -307,7 +307,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -317,8 +317,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -328,17 +328,17 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
@@ -471,8 +471,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -491,27 +491,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul32.c
index 1cfa1dd..16c8cc1 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul32.c
@@ -224,8 +224,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -254,7 +254,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -264,8 +264,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -275,12 +275,12 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -351,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -364,21 +364,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-ssse3-mul16.c
index ed0f6d7..5229fe7 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-ssse3-mul16.c
@@ -277,8 +277,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -344,7 +344,7 @@
const __m128i vq31prod89AB = _mm_shuffle_epi32(vq31prod8A9B, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prodCDEF = _mm_shuffle_epi32(vq31prodCEDF, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -354,8 +354,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -365,15 +365,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
@@ -510,8 +510,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -550,24 +550,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-wasmsimd-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-wasmsimd-mul16.c
index 04f51d4..2887a36 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-wasmsimd-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-wasmsimd-mul16.c
@@ -228,8 +228,8 @@
const v128_t vaccCD = wasm_v32x4_shuffle(vaccCDEF, vsignCDEF, 0, 4, 1, 5);
const v128_t vaccEF = wasm_v32x4_shuffle(vaccCDEF, vsignCDEF, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -244,25 +244,25 @@
const v128_t vq31prod89AB = wasm_v32x4_shuffle(vprod89, vprodAB, 1, 3, 5, 7);
const v128_t vq31prodCDEF = wasm_v32x4_shuffle(vprodCD, vprodEF, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
const v128_t vrem89AB = wasm_i32x4_add(wasm_v128_and(vq31prod89AB, vremainder_mask), wasm_i32x4_shr(vq31prod89AB, 31));
const v128_t vremCDEF = wasm_i32x4_add(wasm_v128_and(vq31prodCDEF, vremainder_mask), wasm_i32x4_shr(vq31prodCDEF, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
vacc89AB = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod89AB, vshift), wasm_i32x4_gt(vrem89AB, vthreshold));
vaccCDEF = wasm_i32x4_sub(wasm_i32x4_shr(vq31prodCDEF, vshift), wasm_i32x4_gt(vremCDEF, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
v128_t vout89ABCDEF = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456789ABCDEF = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF), voutput_min), voutput_max);
wasm_v128_store(output, vout0123456789ABCDEF);
@@ -367,8 +367,8 @@
const v128_t vacc45 = wasm_v32x4_shuffle(vacc4567, vsign4567, 0, 4, 1, 5);
const v128_t vacc67 = wasm_v32x4_shuffle(vacc4567, vsign4567, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -377,20 +377,20 @@
const v128_t vq31prod0123 = wasm_v32x4_shuffle(vprod01, vprod23, 1, 3, 5, 7);
const v128_t vq31prod4567 = wasm_v32x4_shuffle(vprod45, vprod67, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
if XNN_LIKELY(c >= 8) {
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-xop-mul32.c
index 5dba590..21573ca 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-xop-mul32.c
@@ -229,8 +229,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -259,7 +259,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -269,8 +269,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -280,12 +280,12 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -356,8 +356,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -369,21 +369,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up1x25-minmax-gemmlowp-scalar.c b/src/qs8-dwconv/gen/up1x25-minmax-gemmlowp-scalar.c
index 2923490..f472472 100644
--- a/src/qs8-dwconv/gen/up1x25-minmax-gemmlowp-scalar.c
+++ b/src/qs8-dwconv/gen/up1x25-minmax-gemmlowp-scalar.c
@@ -28,14 +28,14 @@
assert(channels != 0);
assert(output_width != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vq31rounding = INT64_C(0x40000000);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up1x9-minmax-gemmlowp-scalar.c b/src/qs8-dwconv/gen/up1x9-minmax-gemmlowp-scalar.c
index 9980583..0279786 100644
--- a/src/qs8-dwconv/gen/up1x9-minmax-gemmlowp-scalar.c
+++ b/src/qs8-dwconv/gen/up1x9-minmax-gemmlowp-scalar.c
@@ -28,14 +28,14 @@
assert(channels != 0);
assert(output_width != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vq31rounding = INT64_C(0x40000000);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx2-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx2-mul32.c
new file mode 100644
index 0000000..08f6a46
--- /dev/null
+++ b/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx2-mul32.c
@@ -0,0 +1,699 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul32.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up24x25__avx2_mul32(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 24; c -= 24) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+ __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
+ const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+ const __m256i vi0xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 16)));
+ const __m256i vk0xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ i0 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi0xGHIJKLMN, vk0xGHIJKLMN));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+ const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
+ const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ const __m256i vi1xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 16)));
+ const __m256i vk1xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+ i1 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi1xGHIJKLMN, vk1xGHIJKLMN));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
+ const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+ const __m256i vi2xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 16)));
+ const __m256i vk2xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ i2 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi2xGHIJKLMN, vk2xGHIJKLMN));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 72 * sizeof(int8_t))));
+ const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
+ const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ const __m256i vi3xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 16)));
+ const __m256i vk3xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t))));
+ i3 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi3xGHIJKLMN, vk3xGHIJKLMN));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
+ const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 104 * sizeof(int8_t))));
+ const __m256i vi4xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 16)));
+ const __m256i vk4xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ i4 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi4xGHIJKLMN, vk4xGHIJKLMN));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 120 * sizeof(int8_t))));
+ const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
+ const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ const __m256i vi5xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 16)));
+ const __m256i vk5xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t))));
+ i5 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi5xGHIJKLMN, vk5xGHIJKLMN));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
+ const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 152 * sizeof(int8_t))));
+ const __m256i vi6xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 16)));
+ const __m256i vk6xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ i6 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi6xGHIJKLMN, vk6xGHIJKLMN));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 168 * sizeof(int8_t))));
+ const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
+ const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ const __m256i vi7xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 16)));
+ const __m256i vk7xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t))));
+ i7 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi7xGHIJKLMN, vk7xGHIJKLMN));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ const __m256i vi8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 8)));
+ const __m256i vk8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 200 * sizeof(int8_t))));
+ const __m256i vi8xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 16)));
+ const __m256i vk8xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+ i8 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi8x89ABCDEF, vk8x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi8xGHIJKLMN, vk8xGHIJKLMN));
+
+ const __m256i vi9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i9));
+ const __m256i vk9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t))));
+ const __m256i vi9x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i9 + 8)));
+ const __m256i vk9x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 224 * sizeof(int8_t))));
+ const __m256i vi9xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i9 + 16)));
+ const __m256i vk9xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 232 * sizeof(int8_t))));
+ i9 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi9x01234567, vk9x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi9x89ABCDEF, vk9x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi9xGHIJKLMN, vk9xGHIJKLMN));
+
+ const __m256i vi10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i10));
+ const __m256i vk10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 240 * sizeof(int8_t))));
+ const __m256i vi10x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i10 + 8)));
+ const __m256i vk10x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 248 * sizeof(int8_t))));
+ const __m256i vi10xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i10 + 16)));
+ const __m256i vk10xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 256 * sizeof(int8_t))));
+ i10 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi10x01234567, vk10x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi10x89ABCDEF, vk10x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi10xGHIJKLMN, vk10xGHIJKLMN));
+
+ const __m256i vi11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i11));
+ const __m256i vk11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 264 * sizeof(int8_t))));
+ const __m256i vi11x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i11 + 8)));
+ const __m256i vk11x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 272 * sizeof(int8_t))));
+ const __m256i vi11xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i11 + 16)));
+ const __m256i vk11xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 280 * sizeof(int8_t))));
+ i11 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi11x01234567, vk11x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi11x89ABCDEF, vk11x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi11xGHIJKLMN, vk11xGHIJKLMN));
+
+ const __m256i vi12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i12));
+ const __m256i vk12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 288 * sizeof(int8_t))));
+ const __m256i vi12x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i12 + 8)));
+ const __m256i vk12x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 296 * sizeof(int8_t))));
+ const __m256i vi12xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i12 + 16)));
+ const __m256i vk12xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 304 * sizeof(int8_t))));
+ i12 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi12x01234567, vk12x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi12x89ABCDEF, vk12x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi12xGHIJKLMN, vk12xGHIJKLMN));
+
+ const __m256i vi13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i13));
+ const __m256i vk13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 312 * sizeof(int8_t))));
+ const __m256i vi13x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i13 + 8)));
+ const __m256i vk13x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 320 * sizeof(int8_t))));
+ const __m256i vi13xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i13 + 16)));
+ const __m256i vk13xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 328 * sizeof(int8_t))));
+ i13 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi13x01234567, vk13x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi13x89ABCDEF, vk13x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi13xGHIJKLMN, vk13xGHIJKLMN));
+
+ const __m256i vi14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i14));
+ const __m256i vk14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 336 * sizeof(int8_t))));
+ const __m256i vi14x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i14 + 8)));
+ const __m256i vk14x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 344 * sizeof(int8_t))));
+ const __m256i vi14xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i14 + 16)));
+ const __m256i vk14xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 352 * sizeof(int8_t))));
+ i14 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi14x01234567, vk14x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi14x89ABCDEF, vk14x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi14xGHIJKLMN, vk14xGHIJKLMN));
+
+ const __m256i vi15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i15));
+ const __m256i vk15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 360 * sizeof(int8_t))));
+ const __m256i vi15x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i15 + 8)));
+ const __m256i vk15x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 368 * sizeof(int8_t))));
+ const __m256i vi15xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i15 + 16)));
+ const __m256i vk15xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 376 * sizeof(int8_t))));
+ i15 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi15x01234567, vk15x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi15x89ABCDEF, vk15x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi15xGHIJKLMN, vk15xGHIJKLMN));
+
+ const __m256i vi16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i16));
+ const __m256i vk16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 384 * sizeof(int8_t))));
+ const __m256i vi16x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i16 + 8)));
+ const __m256i vk16x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 392 * sizeof(int8_t))));
+ const __m256i vi16xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i16 + 16)));
+ const __m256i vk16xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 400 * sizeof(int8_t))));
+ i16 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi16x01234567, vk16x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi16x89ABCDEF, vk16x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi16xGHIJKLMN, vk16xGHIJKLMN));
+
+ const __m256i vi17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i17));
+ const __m256i vk17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 408 * sizeof(int8_t))));
+ const __m256i vi17x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i17 + 8)));
+ const __m256i vk17x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 416 * sizeof(int8_t))));
+ const __m256i vi17xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i17 + 16)));
+ const __m256i vk17xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 424 * sizeof(int8_t))));
+ i17 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi17x01234567, vk17x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi17x89ABCDEF, vk17x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi17xGHIJKLMN, vk17xGHIJKLMN));
+
+ const __m256i vi18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i18));
+ const __m256i vk18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 432 * sizeof(int8_t))));
+ const __m256i vi18x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i18 + 8)));
+ const __m256i vk18x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 440 * sizeof(int8_t))));
+ const __m256i vi18xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i18 + 16)));
+ const __m256i vk18xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 448 * sizeof(int8_t))));
+ i18 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi18x01234567, vk18x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi18x89ABCDEF, vk18x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi18xGHIJKLMN, vk18xGHIJKLMN));
+
+ const __m256i vi19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i19));
+ const __m256i vk19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 456 * sizeof(int8_t))));
+ const __m256i vi19x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i19 + 8)));
+ const __m256i vk19x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 464 * sizeof(int8_t))));
+ const __m256i vi19xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i19 + 16)));
+ const __m256i vk19xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 472 * sizeof(int8_t))));
+ i19 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi19x01234567, vk19x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi19x89ABCDEF, vk19x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi19xGHIJKLMN, vk19xGHIJKLMN));
+
+ const __m256i vi20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i20));
+ const __m256i vk20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 480 * sizeof(int8_t))));
+ const __m256i vi20x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i20 + 8)));
+ const __m256i vk20x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 488 * sizeof(int8_t))));
+ const __m256i vi20xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i20 + 16)));
+ const __m256i vk20xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 496 * sizeof(int8_t))));
+ i20 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi20x01234567, vk20x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi20x89ABCDEF, vk20x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi20xGHIJKLMN, vk20xGHIJKLMN));
+
+ const __m256i vi21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i21));
+ const __m256i vk21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 504 * sizeof(int8_t))));
+ const __m256i vi21x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i21 + 8)));
+ const __m256i vk21x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 512 * sizeof(int8_t))));
+ const __m256i vi21xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i21 + 16)));
+ const __m256i vk21xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 520 * sizeof(int8_t))));
+ i21 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi21x01234567, vk21x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi21x89ABCDEF, vk21x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi21xGHIJKLMN, vk21xGHIJKLMN));
+
+ const __m256i vi22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i22));
+ const __m256i vk22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 528 * sizeof(int8_t))));
+ const __m256i vi22x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i22 + 8)));
+ const __m256i vk22x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 536 * sizeof(int8_t))));
+ const __m256i vi22xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i22 + 16)));
+ const __m256i vk22xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 544 * sizeof(int8_t))));
+ i22 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi22x01234567, vk22x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi22x89ABCDEF, vk22x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi22xGHIJKLMN, vk22xGHIJKLMN));
+
+ const __m256i vi23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i23));
+ const __m256i vk23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 552 * sizeof(int8_t))));
+ const __m256i vi23x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i23 + 8)));
+ const __m256i vk23x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 560 * sizeof(int8_t))));
+ const __m256i vi23xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i23 + 16)));
+ const __m256i vk23xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 568 * sizeof(int8_t))));
+ i23 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi23x01234567, vk23x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi23x89ABCDEF, vk23x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi23xGHIJKLMN, vk23xGHIJKLMN));
+
+ const __m256i vi24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i24));
+ const __m256i vk24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 576 * sizeof(int8_t))));
+ const __m256i vi24x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i24 + 8)));
+ const __m256i vk24x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 584 * sizeof(int8_t))));
+ const __m256i vi24xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i24 + 16)));
+ const __m256i vk24xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 592 * sizeof(int8_t))));
+ i24 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi24x89ABCDEF, vk24x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi24xGHIJKLMN, vk24xGHIJKLMN));
+
+ w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+ __m256 vscaledGHIJKLMN = _mm256_cvtepi32_ps(vaccGHIJKLMN);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+ vscaledGHIJKLMN = _mm256_mul_ps(vscaledGHIJKLMN, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+ vaccGHIJKLMN = _mm256_cvtps_epi32(vscaledGHIJKLMN);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+ __m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vaccGHIJKLMN), _mm256_extracti128_si256(vaccGHIJKLMN, 1)), _mm256_castsi256_si128(voutput_zero_point));
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ __m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ _mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
+ output += 24;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ do {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) k));
+ i0 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 24)));
+ i1 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 48)));
+ i2 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 72)));
+ i3 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 96)));
+ i4 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 120)));
+ i5 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 144)));
+ i6 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 168)));
+ i7 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 192)));
+ i8 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ const __m256i vi9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i9));
+ const __m256i vk9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 216)));
+ i9 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi9x01234567, vk9x01234567));
+
+ const __m256i vi10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i10));
+ const __m256i vk10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 240)));
+ i10 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi10x01234567, vk10x01234567));
+
+ const __m256i vi11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i11));
+ const __m256i vk11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 264)));
+ i11 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi11x01234567, vk11x01234567));
+
+ const __m256i vi12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i12));
+ const __m256i vk12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 288)));
+ i12 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi12x01234567, vk12x01234567));
+
+ const __m256i vi13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i13));
+ const __m256i vk13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 312)));
+ i13 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi13x01234567, vk13x01234567));
+
+ const __m256i vi14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i14));
+ const __m256i vk14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 336)));
+ i14 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi14x01234567, vk14x01234567));
+
+ const __m256i vi15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i15));
+ const __m256i vk15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 360)));
+ i15 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi15x01234567, vk15x01234567));
+
+ const __m256i vi16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i16));
+ const __m256i vk16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 384)));
+ i16 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi16x01234567, vk16x01234567));
+
+ const __m256i vi17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i17));
+ const __m256i vk17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 408)));
+ i17 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi17x01234567, vk17x01234567));
+
+ const __m256i vi18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i18));
+ const __m256i vk18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 432)));
+ i18 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi18x01234567, vk18x01234567));
+
+ const __m256i vi19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i19));
+ const __m256i vk19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 456)));
+ i19 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi19x01234567, vk19x01234567));
+
+ const __m256i vi20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i20));
+ const __m256i vk20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 480)));
+ i20 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi20x01234567, vk20x01234567));
+
+ const __m256i vi21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i21));
+ const __m256i vk21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 504)));
+ i21 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi21x01234567, vk21x01234567));
+
+ const __m256i vi22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i22));
+ const __m256i vk22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 528)));
+ i22 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi22x01234567, vk22x01234567));
+
+ const __m256i vi23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i23));
+ const __m256i vk23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 552)));
+ i23 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi23x01234567, vk23x01234567));
+
+ const __m256i vi24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i24));
+ const __m256i vk24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 576)));
+ i24 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
+
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ k += 8;
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ if XNN_LIKELY(c >= 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456701234567);
+ output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
+ vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456701234567, 0);
+ vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
+ output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c
index 8da8409..6316096 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c
@@ -895,8 +895,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -937,7 +937,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -951,8 +951,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -966,7 +966,7 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
@@ -975,11 +975,11 @@
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
@@ -1322,8 +1322,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -1342,27 +1342,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c
index 56ef1a7..f676a89 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c
@@ -696,8 +696,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -738,7 +738,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -752,8 +752,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -767,13 +767,13 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -944,8 +944,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -957,21 +957,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx2-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx2-mul32.c
index 1b127a2..bf2dab7 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx2-mul32.c
@@ -467,8 +467,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -492,7 +492,7 @@
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -500,8 +500,8 @@
const __m256i vremGHIJKLMN =
_mm256_add_epi32(_mm256_and_si256(vq31prodGHIJKLMN, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodGHIJKLMN));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -509,15 +509,15 @@
vaccGHIJKLMN =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodGHIJKLMN, vshift), _mm256_cmpgt_epi32(vremGHIJKLMN, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vaccGHIJKLMN), _mm256_extracti128_si256(vaccGHIJKLMN, 1)), _mm256_castsi256_si128(voutput_zero_point));
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
@@ -686,8 +686,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -699,21 +699,21 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-neon-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-neon-mul16.c
index 8644556..da8766f 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-neon-mul16.c
@@ -29,12 +29,12 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse2-mul16.c
index ed180a1..52f6f3d 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse2-mul16.c
@@ -895,8 +895,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -990,7 +990,7 @@
const __m128i vq31prodGHIJ = _mm_shuffle_epi32(vq31prodGIHJ, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prodKLMN = _mm_shuffle_epi32(vq31prodKMLN, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -1004,8 +1004,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -1019,17 +1019,17 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
voutGHIJKLMN = _mm_max_epi16(voutGHIJKLMN, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
voutGHIJKLMN = _mm_min_epi16(voutGHIJKLMN, voutput_max);
@@ -1377,8 +1377,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -1417,24 +1417,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul16.c
index bffa0c6..18ee6a7 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul16.c
@@ -895,8 +895,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -937,7 +937,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -951,8 +951,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -966,7 +966,7 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
@@ -975,11 +975,11 @@
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
@@ -1322,8 +1322,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -1342,27 +1342,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul32.c
index c0b7cc5..59a5fcd 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul32.c
@@ -696,8 +696,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -738,7 +738,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -752,8 +752,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -767,13 +767,13 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -944,8 +944,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -957,21 +957,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-ssse3-mul16.c
index 0262f2a..c3efb42 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-ssse3-mul16.c
@@ -895,8 +895,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -990,7 +990,7 @@
const __m128i vq31prodGHIJ = _mm_shuffle_epi32(vq31prodGIHJ, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prodKLMN = _mm_shuffle_epi32(vq31prodKMLN, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -1004,8 +1004,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -1019,17 +1019,17 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
voutGHIJKLMN = _mm_max_epi16(voutGHIJKLMN, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
voutGHIJKLMN = _mm_min_epi16(voutGHIJKLMN, voutput_max);
@@ -1377,8 +1377,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -1417,24 +1417,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-wasmsimd-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-wasmsimd-mul16.c
index f52a48f..3b9667d 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-wasmsimd-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-wasmsimd-mul16.c
@@ -665,8 +665,8 @@
const v128_t vaccKL = wasm_v32x4_shuffle(vaccKLMN, vsignKLMN, 0, 4, 1, 5);
const v128_t vaccMN = wasm_v32x4_shuffle(vaccKLMN, vsignKLMN, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -687,7 +687,7 @@
const v128_t vq31prodGHIJ = wasm_v32x4_shuffle(vprodGH, vprodIJ, 1, 3, 5, 7);
const v128_t vq31prodKLMN = wasm_v32x4_shuffle(vprodKL, vprodMN, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
const v128_t vrem89AB = wasm_i32x4_add(wasm_v128_and(vq31prod89AB, vremainder_mask), wasm_i32x4_shr(vq31prod89AB, 31));
@@ -695,8 +695,8 @@
const v128_t vremGHIJ = wasm_i32x4_add(wasm_v128_and(vq31prodGHIJ, vremainder_mask), wasm_i32x4_shr(vq31prodGHIJ, 31));
const v128_t vremKLMN = wasm_i32x4_add(wasm_v128_and(vq31prodKLMN, vremainder_mask), wasm_i32x4_shr(vq31prodKLMN, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
vacc89AB = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod89AB, vshift), wasm_i32x4_gt(vrem89AB, vthreshold));
@@ -704,13 +704,13 @@
vaccGHIJ = wasm_i32x4_sub(wasm_i32x4_shr(vq31prodGHIJ, vshift), wasm_i32x4_gt(vremGHIJ, vthreshold));
vaccKLMN = wasm_i32x4_sub(wasm_i32x4_shr(vq31prodKLMN, vshift), wasm_i32x4_gt(vremKLMN, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
v128_t vout89ABCDEF = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF), voutput_zero_point);
v128_t voutGHIJKLMN = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456789ABCDEF = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF), voutput_min), voutput_max);
v128_t voutGHIJKLMNGHIJKLMN = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(voutGHIJKLMN, voutGHIJKLMN), voutput_min), voutput_max);
@@ -961,8 +961,8 @@
const v128_t vacc45 = wasm_v32x4_shuffle(vacc4567, vsign4567, 0, 4, 1, 5);
const v128_t vacc67 = wasm_v32x4_shuffle(vacc4567, vsign4567, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -971,20 +971,20 @@
const v128_t vq31prod0123 = wasm_v32x4_shuffle(vprod01, vprod23, 1, 3, 5, 7);
const v128_t vq31prod4567 = wasm_v32x4_shuffle(vprod45, vprod67, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
if XNN_LIKELY(c >= 8) {
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c
index cc184a9..a7caee2 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c
@@ -701,8 +701,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -743,7 +743,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -757,8 +757,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -772,13 +772,13 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -949,8 +949,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -962,21 +962,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx2-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx2-mul32.c
new file mode 100644
index 0000000..c10fa47
--- /dev/null
+++ b/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx2-mul32.c
@@ -0,0 +1,331 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul32.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up24x9__avx2_mul32(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 24; c -= 24) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+ __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
+ const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+ const __m256i vi0xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 16)));
+ const __m256i vk0xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ i0 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi0xGHIJKLMN, vk0xGHIJKLMN));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+ const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
+ const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ const __m256i vi1xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 16)));
+ const __m256i vk1xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+ i1 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi1xGHIJKLMN, vk1xGHIJKLMN));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
+ const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+ const __m256i vi2xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 16)));
+ const __m256i vk2xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ i2 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi2xGHIJKLMN, vk2xGHIJKLMN));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 72 * sizeof(int8_t))));
+ const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
+ const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ const __m256i vi3xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 16)));
+ const __m256i vk3xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t))));
+ i3 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi3xGHIJKLMN, vk3xGHIJKLMN));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
+ const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 104 * sizeof(int8_t))));
+ const __m256i vi4xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 16)));
+ const __m256i vk4xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ i4 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi4xGHIJKLMN, vk4xGHIJKLMN));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 120 * sizeof(int8_t))));
+ const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
+ const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ const __m256i vi5xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 16)));
+ const __m256i vk5xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t))));
+ i5 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi5xGHIJKLMN, vk5xGHIJKLMN));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
+ const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 152 * sizeof(int8_t))));
+ const __m256i vi6xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 16)));
+ const __m256i vk6xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ i6 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi6xGHIJKLMN, vk6xGHIJKLMN));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 168 * sizeof(int8_t))));
+ const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
+ const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ const __m256i vi7xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 16)));
+ const __m256i vk7xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t))));
+ i7 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi7xGHIJKLMN, vk7xGHIJKLMN));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ const __m256i vi8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 8)));
+ const __m256i vk8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 200 * sizeof(int8_t))));
+ const __m256i vi8xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 16)));
+ const __m256i vk8xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+ i8 += 24;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi8x89ABCDEF, vk8x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi8xGHIJKLMN, vk8xGHIJKLMN));
+
+ w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+ __m256 vscaledGHIJKLMN = _mm256_cvtepi32_ps(vaccGHIJKLMN);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+ vscaledGHIJKLMN = _mm256_mul_ps(vscaledGHIJKLMN, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+ vaccGHIJKLMN = _mm256_cvtps_epi32(vscaledGHIJKLMN);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+ __m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vaccGHIJKLMN), _mm256_extracti128_si256(vaccGHIJKLMN, 1)), _mm256_castsi256_si128(voutput_zero_point));
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ __m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ _mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
+ output += 24;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ do {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) k));
+ i0 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 24)));
+ i1 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 48)));
+ i2 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 72)));
+ i3 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 96)));
+ i4 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 120)));
+ i5 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 144)));
+ i6 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 168)));
+ i7 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 192)));
+ i8 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ k += 8;
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ if XNN_LIKELY(c >= 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456701234567);
+ output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
+ vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456701234567, 0);
+ vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
+ output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul16.c
index 4b34be0..4e6fe18 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul16.c
@@ -351,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -393,7 +393,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -407,8 +407,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -422,7 +422,7 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
@@ -431,11 +431,11 @@
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
@@ -570,8 +570,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -590,27 +590,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c
index f96e89d..027dbbe 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c
@@ -280,8 +280,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -322,7 +322,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -336,8 +336,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -351,13 +351,13 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -432,8 +432,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -445,21 +445,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx2-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx2-mul32.c
index 9642bde..1fe0d8d 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx2-mul32.c
@@ -195,8 +195,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -220,7 +220,7 @@
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -228,8 +228,8 @@
const __m256i vremGHIJKLMN =
_mm256_add_epi32(_mm256_and_si256(vq31prodGHIJKLMN, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodGHIJKLMN));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -237,15 +237,15 @@
vaccGHIJKLMN =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodGHIJKLMN, vshift), _mm256_cmpgt_epi32(vremGHIJKLMN, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vaccGHIJKLMN), _mm256_extracti128_si256(vaccGHIJKLMN, 1)), _mm256_castsi256_si128(voutput_zero_point));
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
@@ -318,8 +318,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -331,21 +331,21 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-neon-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-neon-mul16.c
index fa108eb..97998c7 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-neon-mul16.c
@@ -29,12 +29,12 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse2-mul16.c
index e0526bf..320dc68 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse2-mul16.c
@@ -351,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -446,7 +446,7 @@
const __m128i vq31prodGHIJ = _mm_shuffle_epi32(vq31prodGIHJ, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prodKLMN = _mm_shuffle_epi32(vq31prodKMLN, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -460,8 +460,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -475,17 +475,17 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
voutGHIJKLMN = _mm_max_epi16(voutGHIJKLMN, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
voutGHIJKLMN = _mm_min_epi16(voutGHIJKLMN, voutput_max);
@@ -625,8 +625,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -665,24 +665,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul16.c
index 3d3ff1f..ac66d22 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul16.c
@@ -351,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -393,7 +393,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -407,8 +407,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -422,7 +422,7 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
@@ -431,11 +431,11 @@
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
@@ -570,8 +570,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -590,27 +590,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul32.c
index 446dfb8..d56b5ba 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul32.c
@@ -280,8 +280,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -322,7 +322,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -336,8 +336,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -351,13 +351,13 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -432,8 +432,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -445,21 +445,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-ssse3-mul16.c
index 4b723de..9a9d8ee 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-ssse3-mul16.c
@@ -351,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -446,7 +446,7 @@
const __m128i vq31prodGHIJ = _mm_shuffle_epi32(vq31prodGIHJ, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prodKLMN = _mm_shuffle_epi32(vq31prodKMLN, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -460,8 +460,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -475,17 +475,17 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
voutGHIJKLMN = _mm_max_epi16(voutGHIJKLMN, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
voutGHIJKLMN = _mm_min_epi16(voutGHIJKLMN, voutput_max);
@@ -625,8 +625,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -665,24 +665,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-wasmsimd-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-wasmsimd-mul16.c
index 5b048d9..1f5fc62 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-wasmsimd-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-wasmsimd-mul16.c
@@ -281,8 +281,8 @@
const v128_t vaccKL = wasm_v32x4_shuffle(vaccKLMN, vsignKLMN, 0, 4, 1, 5);
const v128_t vaccMN = wasm_v32x4_shuffle(vaccKLMN, vsignKLMN, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -303,7 +303,7 @@
const v128_t vq31prodGHIJ = wasm_v32x4_shuffle(vprodGH, vprodIJ, 1, 3, 5, 7);
const v128_t vq31prodKLMN = wasm_v32x4_shuffle(vprodKL, vprodMN, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
const v128_t vrem89AB = wasm_i32x4_add(wasm_v128_and(vq31prod89AB, vremainder_mask), wasm_i32x4_shr(vq31prod89AB, 31));
@@ -311,8 +311,8 @@
const v128_t vremGHIJ = wasm_i32x4_add(wasm_v128_and(vq31prodGHIJ, vremainder_mask), wasm_i32x4_shr(vq31prodGHIJ, 31));
const v128_t vremKLMN = wasm_i32x4_add(wasm_v128_and(vq31prodKLMN, vremainder_mask), wasm_i32x4_shr(vq31prodKLMN, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
vacc89AB = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod89AB, vshift), wasm_i32x4_gt(vrem89AB, vthreshold));
@@ -320,13 +320,13 @@
vaccGHIJ = wasm_i32x4_sub(wasm_i32x4_shr(vq31prodGHIJ, vshift), wasm_i32x4_gt(vremGHIJ, vthreshold));
vaccKLMN = wasm_i32x4_sub(wasm_i32x4_shr(vq31prodKLMN, vshift), wasm_i32x4_gt(vremKLMN, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
v128_t vout89ABCDEF = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF), voutput_zero_point);
v128_t voutGHIJKLMN = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456789ABCDEF = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF), voutput_min), voutput_max);
v128_t voutGHIJKLMNGHIJKLMN = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(voutGHIJKLMN, voutGHIJKLMN), voutput_min), voutput_max);
@@ -433,8 +433,8 @@
const v128_t vacc45 = wasm_v32x4_shuffle(vacc4567, vsign4567, 0, 4, 1, 5);
const v128_t vacc67 = wasm_v32x4_shuffle(vacc4567, vsign4567, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -443,20 +443,20 @@
const v128_t vq31prod0123 = wasm_v32x4_shuffle(vprod01, vprod23, 1, 3, 5, 7);
const v128_t vq31prod4567 = wasm_v32x4_shuffle(vprod45, vprod67, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
if XNN_LIKELY(c >= 8) {
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c
index a3e631f..752626a 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c
@@ -285,8 +285,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -327,7 +327,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -341,8 +341,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -356,13 +356,13 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
@@ -437,8 +437,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -450,21 +450,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up2x25-minmax-gemmlowp-scalar.c b/src/qs8-dwconv/gen/up2x25-minmax-gemmlowp-scalar.c
index fb742e7..37fc8d7 100644
--- a/src/qs8-dwconv/gen/up2x25-minmax-gemmlowp-scalar.c
+++ b/src/qs8-dwconv/gen/up2x25-minmax-gemmlowp-scalar.c
@@ -28,14 +28,14 @@
assert(channels != 0);
assert(output_width != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vq31rounding = INT64_C(0x40000000);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up2x9-minmax-gemmlowp-scalar.c b/src/qs8-dwconv/gen/up2x9-minmax-gemmlowp-scalar.c
index ae2a2a3..e7ed15b 100644
--- a/src/qs8-dwconv/gen/up2x9-minmax-gemmlowp-scalar.c
+++ b/src/qs8-dwconv/gen/up2x9-minmax-gemmlowp-scalar.c
@@ -28,14 +28,14 @@
assert(channels != 0);
assert(output_width != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vq31rounding = INT64_C(0x40000000);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-fp32-avx2-mul16.c b/src/qs8-dwconv/gen/up32x25-minmax-fp32-avx2-mul16.c
new file mode 100644
index 0000000..5fbbc91
--- /dev/null
+++ b/src/qs8-dwconv/gen/up32x25-minmax-fp32-avx2-mul16.c
@@ -0,0 +1,865 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul16.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up32x25__avx2_mul16(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 32; c -= 32) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+ __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
+ __m256i vaccOPQRSTUV = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 24 * sizeof(int32_t)));
+
+
+ const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
+ const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ const __m256i vi0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i0 + 16)));
+ const __m256i vk0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ i0 += 32;
+
+ const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
+ const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
+ const __m256i vprod0xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi0xGHIJKLMNOPQRSTUV, vk0xGHIJKLMNOPQRSTUV);
+ const __m128i vprod0xOPQRSTUV = _mm256_extracti128_si256(vprod0xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod0xOPQRSTUV));
+
+ const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
+ const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ const __m256i vi1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i1 + 16)));
+ const __m256i vk1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ i1 += 32;
+
+ const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
+ const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
+ const __m256i vprod1xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi1xGHIJKLMNOPQRSTUV, vk1xGHIJKLMNOPQRSTUV);
+ const __m128i vprod1xOPQRSTUV = _mm256_extracti128_si256(vprod1xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod1xOPQRSTUV));
+
+ const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
+ const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ const __m256i vi2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i2 + 16)));
+ const __m256i vk2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ i2 += 32;
+
+ const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
+ const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
+ const __m256i vprod2xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi2xGHIJKLMNOPQRSTUV, vk2xGHIJKLMNOPQRSTUV);
+ const __m128i vprod2xOPQRSTUV = _mm256_extracti128_si256(vprod2xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod2xOPQRSTUV));
+
+ const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
+ const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ const __m256i vi3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i3 + 16)));
+ const __m256i vk3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ i3 += 32;
+
+ const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
+ const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
+ const __m256i vprod3xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi3xGHIJKLMNOPQRSTUV, vk3xGHIJKLMNOPQRSTUV);
+ const __m128i vprod3xOPQRSTUV = _mm256_extracti128_si256(vprod3xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod3xOPQRSTUV));
+
+ const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
+ const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ const __m256i vi4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i4 + 16)));
+ const __m256i vk4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ i4 += 32;
+
+ const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
+ const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
+ const __m256i vprod4xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi4xGHIJKLMNOPQRSTUV, vk4xGHIJKLMNOPQRSTUV);
+ const __m128i vprod4xOPQRSTUV = _mm256_extracti128_si256(vprod4xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod4xOPQRSTUV));
+
+ const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
+ const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ const __m256i vi5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i5 + 16)));
+ const __m256i vk5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ i5 += 32;
+
+ const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
+ const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
+ const __m256i vprod5xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi5xGHIJKLMNOPQRSTUV, vk5xGHIJKLMNOPQRSTUV);
+ const __m128i vprod5xOPQRSTUV = _mm256_extracti128_si256(vprod5xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod5xOPQRSTUV));
+
+ const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
+ const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ const __m256i vi6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i6 + 16)));
+ const __m256i vk6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+ i6 += 32;
+
+ const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
+ const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
+ const __m256i vprod6xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi6xGHIJKLMNOPQRSTUV, vk6xGHIJKLMNOPQRSTUV);
+ const __m128i vprod6xOPQRSTUV = _mm256_extracti128_si256(vprod6xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod6xOPQRSTUV));
+
+ const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
+ const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 224 * sizeof(int8_t))));
+ const __m256i vi7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i7 + 16)));
+ const __m256i vk7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 240 * sizeof(int8_t))));
+ i7 += 32;
+
+ const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
+ const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
+ const __m256i vprod7xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi7xGHIJKLMNOPQRSTUV, vk7xGHIJKLMNOPQRSTUV);
+ const __m128i vprod7xOPQRSTUV = _mm256_extracti128_si256(vprod7xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod7xOPQRSTUV));
+
+ const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
+ const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 256 * sizeof(int8_t))));
+ const __m256i vi8xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i8 + 16)));
+ const __m256i vk8xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 272 * sizeof(int8_t))));
+ i8 += 32;
+
+ const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
+ const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
+ const __m256i vprod8xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi8xGHIJKLMNOPQRSTUV, vk8xGHIJKLMNOPQRSTUV);
+ const __m128i vprod8xOPQRSTUV = _mm256_extracti128_si256(vprod8xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod8xOPQRSTUV));
+
+ const __m256i vi9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i9));
+ const __m256i vk9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t))));
+ const __m256i vi9xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i9 + 16)));
+ const __m256i vk9xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 304 * sizeof(int8_t))));
+ i9 += 32;
+
+ const __m256i vprod9x0123456789ABCDEF = _mm256_mullo_epi16(vi9x0123456789ABCDEF, vk9x0123456789ABCDEF);
+ const __m128i vprod9x89ABCDEF = _mm256_extracti128_si256(vprod9x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod9x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod9x89ABCDEF));
+ const __m256i vprod9xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi9xGHIJKLMNOPQRSTUV, vk9xGHIJKLMNOPQRSTUV);
+ const __m128i vprod9xOPQRSTUV = _mm256_extracti128_si256(vprod9xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod9xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod9xOPQRSTUV));
+
+ const __m256i vi10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i10));
+ const __m256i vk10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 320 * sizeof(int8_t))));
+ const __m256i vi10xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i10 + 16)));
+ const __m256i vk10xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 336 * sizeof(int8_t))));
+ i10 += 32;
+
+ const __m256i vprod10x0123456789ABCDEF = _mm256_mullo_epi16(vi10x0123456789ABCDEF, vk10x0123456789ABCDEF);
+ const __m128i vprod10x89ABCDEF = _mm256_extracti128_si256(vprod10x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod10x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod10x89ABCDEF));
+ const __m256i vprod10xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi10xGHIJKLMNOPQRSTUV, vk10xGHIJKLMNOPQRSTUV);
+ const __m128i vprod10xOPQRSTUV = _mm256_extracti128_si256(vprod10xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod10xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod10xOPQRSTUV));
+
+ const __m256i vi11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i11));
+ const __m256i vk11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 352 * sizeof(int8_t))));
+ const __m256i vi11xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i11 + 16)));
+ const __m256i vk11xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 368 * sizeof(int8_t))));
+ i11 += 32;
+
+ const __m256i vprod11x0123456789ABCDEF = _mm256_mullo_epi16(vi11x0123456789ABCDEF, vk11x0123456789ABCDEF);
+ const __m128i vprod11x89ABCDEF = _mm256_extracti128_si256(vprod11x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod11x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod11x89ABCDEF));
+ const __m256i vprod11xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi11xGHIJKLMNOPQRSTUV, vk11xGHIJKLMNOPQRSTUV);
+ const __m128i vprod11xOPQRSTUV = _mm256_extracti128_si256(vprod11xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod11xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod11xOPQRSTUV));
+
+ const __m256i vi12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i12));
+ const __m256i vk12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 384 * sizeof(int8_t))));
+ const __m256i vi12xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i12 + 16)));
+ const __m256i vk12xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 400 * sizeof(int8_t))));
+ i12 += 32;
+
+ const __m256i vprod12x0123456789ABCDEF = _mm256_mullo_epi16(vi12x0123456789ABCDEF, vk12x0123456789ABCDEF);
+ const __m128i vprod12x89ABCDEF = _mm256_extracti128_si256(vprod12x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod12x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod12x89ABCDEF));
+ const __m256i vprod12xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi12xGHIJKLMNOPQRSTUV, vk12xGHIJKLMNOPQRSTUV);
+ const __m128i vprod12xOPQRSTUV = _mm256_extracti128_si256(vprod12xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod12xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod12xOPQRSTUV));
+
+ const __m256i vi13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i13));
+ const __m256i vk13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 416 * sizeof(int8_t))));
+ const __m256i vi13xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i13 + 16)));
+ const __m256i vk13xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 432 * sizeof(int8_t))));
+ i13 += 32;
+
+ const __m256i vprod13x0123456789ABCDEF = _mm256_mullo_epi16(vi13x0123456789ABCDEF, vk13x0123456789ABCDEF);
+ const __m128i vprod13x89ABCDEF = _mm256_extracti128_si256(vprod13x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod13x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod13x89ABCDEF));
+ const __m256i vprod13xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi13xGHIJKLMNOPQRSTUV, vk13xGHIJKLMNOPQRSTUV);
+ const __m128i vprod13xOPQRSTUV = _mm256_extracti128_si256(vprod13xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod13xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod13xOPQRSTUV));
+
+ const __m256i vi14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i14));
+ const __m256i vk14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 448 * sizeof(int8_t))));
+ const __m256i vi14xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i14 + 16)));
+ const __m256i vk14xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 464 * sizeof(int8_t))));
+ i14 += 32;
+
+ const __m256i vprod14x0123456789ABCDEF = _mm256_mullo_epi16(vi14x0123456789ABCDEF, vk14x0123456789ABCDEF);
+ const __m128i vprod14x89ABCDEF = _mm256_extracti128_si256(vprod14x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod14x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod14x89ABCDEF));
+ const __m256i vprod14xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi14xGHIJKLMNOPQRSTUV, vk14xGHIJKLMNOPQRSTUV);
+ const __m128i vprod14xOPQRSTUV = _mm256_extracti128_si256(vprod14xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod14xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod14xOPQRSTUV));
+
+ const __m256i vi15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i15));
+ const __m256i vk15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 480 * sizeof(int8_t))));
+ const __m256i vi15xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i15 + 16)));
+ const __m256i vk15xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 496 * sizeof(int8_t))));
+ i15 += 32;
+
+ const __m256i vprod15x0123456789ABCDEF = _mm256_mullo_epi16(vi15x0123456789ABCDEF, vk15x0123456789ABCDEF);
+ const __m128i vprod15x89ABCDEF = _mm256_extracti128_si256(vprod15x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod15x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod15x89ABCDEF));
+ const __m256i vprod15xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi15xGHIJKLMNOPQRSTUV, vk15xGHIJKLMNOPQRSTUV);
+ const __m128i vprod15xOPQRSTUV = _mm256_extracti128_si256(vprod15xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod15xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod15xOPQRSTUV));
+
+ const __m256i vi16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i16));
+ const __m256i vk16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 512 * sizeof(int8_t))));
+ const __m256i vi16xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i16 + 16)));
+ const __m256i vk16xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 528 * sizeof(int8_t))));
+ i16 += 32;
+
+ const __m256i vprod16x0123456789ABCDEF = _mm256_mullo_epi16(vi16x0123456789ABCDEF, vk16x0123456789ABCDEF);
+ const __m128i vprod16x89ABCDEF = _mm256_extracti128_si256(vprod16x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod16x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod16x89ABCDEF));
+ const __m256i vprod16xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi16xGHIJKLMNOPQRSTUV, vk16xGHIJKLMNOPQRSTUV);
+ const __m128i vprod16xOPQRSTUV = _mm256_extracti128_si256(vprod16xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod16xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod16xOPQRSTUV));
+
+ const __m256i vi17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i17));
+ const __m256i vk17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 544 * sizeof(int8_t))));
+ const __m256i vi17xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i17 + 16)));
+ const __m256i vk17xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 560 * sizeof(int8_t))));
+ i17 += 32;
+
+ const __m256i vprod17x0123456789ABCDEF = _mm256_mullo_epi16(vi17x0123456789ABCDEF, vk17x0123456789ABCDEF);
+ const __m128i vprod17x89ABCDEF = _mm256_extracti128_si256(vprod17x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod17x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod17x89ABCDEF));
+ const __m256i vprod17xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi17xGHIJKLMNOPQRSTUV, vk17xGHIJKLMNOPQRSTUV);
+ const __m128i vprod17xOPQRSTUV = _mm256_extracti128_si256(vprod17xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod17xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod17xOPQRSTUV));
+
+ const __m256i vi18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i18));
+ const __m256i vk18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 576 * sizeof(int8_t))));
+ const __m256i vi18xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i18 + 16)));
+ const __m256i vk18xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 592 * sizeof(int8_t))));
+ i18 += 32;
+
+ const __m256i vprod18x0123456789ABCDEF = _mm256_mullo_epi16(vi18x0123456789ABCDEF, vk18x0123456789ABCDEF);
+ const __m128i vprod18x89ABCDEF = _mm256_extracti128_si256(vprod18x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod18x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod18x89ABCDEF));
+ const __m256i vprod18xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi18xGHIJKLMNOPQRSTUV, vk18xGHIJKLMNOPQRSTUV);
+ const __m128i vprod18xOPQRSTUV = _mm256_extracti128_si256(vprod18xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod18xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod18xOPQRSTUV));
+
+ const __m256i vi19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i19));
+ const __m256i vk19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 608 * sizeof(int8_t))));
+ const __m256i vi19xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i19 + 16)));
+ const __m256i vk19xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 624 * sizeof(int8_t))));
+ i19 += 32;
+
+ const __m256i vprod19x0123456789ABCDEF = _mm256_mullo_epi16(vi19x0123456789ABCDEF, vk19x0123456789ABCDEF);
+ const __m128i vprod19x89ABCDEF = _mm256_extracti128_si256(vprod19x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod19x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod19x89ABCDEF));
+ const __m256i vprod19xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi19xGHIJKLMNOPQRSTUV, vk19xGHIJKLMNOPQRSTUV);
+ const __m128i vprod19xOPQRSTUV = _mm256_extracti128_si256(vprod19xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod19xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod19xOPQRSTUV));
+
+ const __m256i vi20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i20));
+ const __m256i vk20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 640 * sizeof(int8_t))));
+ const __m256i vi20xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i20 + 16)));
+ const __m256i vk20xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 656 * sizeof(int8_t))));
+ i20 += 32;
+
+ const __m256i vprod20x0123456789ABCDEF = _mm256_mullo_epi16(vi20x0123456789ABCDEF, vk20x0123456789ABCDEF);
+ const __m128i vprod20x89ABCDEF = _mm256_extracti128_si256(vprod20x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod20x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod20x89ABCDEF));
+ const __m256i vprod20xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi20xGHIJKLMNOPQRSTUV, vk20xGHIJKLMNOPQRSTUV);
+ const __m128i vprod20xOPQRSTUV = _mm256_extracti128_si256(vprod20xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod20xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod20xOPQRSTUV));
+
+ const __m256i vi21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i21));
+ const __m256i vk21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 672 * sizeof(int8_t))));
+ const __m256i vi21xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i21 + 16)));
+ const __m256i vk21xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 688 * sizeof(int8_t))));
+ i21 += 32;
+
+ const __m256i vprod21x0123456789ABCDEF = _mm256_mullo_epi16(vi21x0123456789ABCDEF, vk21x0123456789ABCDEF);
+ const __m128i vprod21x89ABCDEF = _mm256_extracti128_si256(vprod21x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod21x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod21x89ABCDEF));
+ const __m256i vprod21xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi21xGHIJKLMNOPQRSTUV, vk21xGHIJKLMNOPQRSTUV);
+ const __m128i vprod21xOPQRSTUV = _mm256_extracti128_si256(vprod21xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod21xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod21xOPQRSTUV));
+
+ const __m256i vi22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i22));
+ const __m256i vk22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 704 * sizeof(int8_t))));
+ const __m256i vi22xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i22 + 16)));
+ const __m256i vk22xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 720 * sizeof(int8_t))));
+ i22 += 32;
+
+ const __m256i vprod22x0123456789ABCDEF = _mm256_mullo_epi16(vi22x0123456789ABCDEF, vk22x0123456789ABCDEF);
+ const __m128i vprod22x89ABCDEF = _mm256_extracti128_si256(vprod22x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod22x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod22x89ABCDEF));
+ const __m256i vprod22xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi22xGHIJKLMNOPQRSTUV, vk22xGHIJKLMNOPQRSTUV);
+ const __m128i vprod22xOPQRSTUV = _mm256_extracti128_si256(vprod22xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod22xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod22xOPQRSTUV));
+
+ const __m256i vi23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i23));
+ const __m256i vk23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 736 * sizeof(int8_t))));
+ const __m256i vi23xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i23 + 16)));
+ const __m256i vk23xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 752 * sizeof(int8_t))));
+ i23 += 32;
+
+ const __m256i vprod23x0123456789ABCDEF = _mm256_mullo_epi16(vi23x0123456789ABCDEF, vk23x0123456789ABCDEF);
+ const __m128i vprod23x89ABCDEF = _mm256_extracti128_si256(vprod23x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod23x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod23x89ABCDEF));
+ const __m256i vprod23xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi23xGHIJKLMNOPQRSTUV, vk23xGHIJKLMNOPQRSTUV);
+ const __m128i vprod23xOPQRSTUV = _mm256_extracti128_si256(vprod23xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod23xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod23xOPQRSTUV));
+
+ const __m256i vi24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i24));
+ const __m256i vk24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 768 * sizeof(int8_t))));
+ const __m256i vi24xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i24 + 16)));
+ const __m256i vk24xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 784 * sizeof(int8_t))));
+ i24 += 32;
+
+ const __m256i vprod24x0123456789ABCDEF = _mm256_mullo_epi16(vi24x0123456789ABCDEF, vk24x0123456789ABCDEF);
+ const __m128i vprod24x89ABCDEF = _mm256_extracti128_si256(vprod24x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod24x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
+ const __m256i vprod24xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi24xGHIJKLMNOPQRSTUV, vk24xGHIJKLMNOPQRSTUV);
+ const __m128i vprod24xOPQRSTUV = _mm256_extracti128_si256(vprod24xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod24xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod24xOPQRSTUV));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 800 * sizeof(int8_t));
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+ __m256 vscaledGHIJKLMN = _mm256_cvtepi32_ps(vaccGHIJKLMN);
+ __m256 vscaledOPQRSTUV = _mm256_cvtepi32_ps(vaccOPQRSTUV);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+ vscaledGHIJKLMN = _mm256_mul_ps(vscaledGHIJKLMN, vscale);
+ vscaledOPQRSTUV = _mm256_mul_ps(vscaledOPQRSTUV, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+ vaccGHIJKLMN = _mm256_cvtps_epi32(vscaledGHIJKLMN);
+ vaccOPQRSTUV = _mm256_cvtps_epi32(vscaledOPQRSTUV);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+ const __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ __m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
+ voutGHIJKLMNOPQRSTUV = _mm_min_epi8(voutGHIJKLMNOPQRSTUV, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ _mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
+ output += 32;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((uintptr_t) w + 32 * sizeof(int32_t));
+ do {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+
+
+ const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
+ const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) k));
+ i0 += 16;
+
+ const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
+ const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
+
+ const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
+ const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 32)));
+ i1 += 16;
+
+ const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
+ const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
+
+ const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
+ const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 64)));
+ i2 += 16;
+
+ const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
+ const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
+
+ const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
+ const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 96)));
+ i3 += 16;
+
+ const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
+ const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
+
+ const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
+ const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 128)));
+ i4 += 16;
+
+ const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
+ const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
+
+ const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
+ const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 160)));
+ i5 += 16;
+
+ const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
+ const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
+
+ const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
+ const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 192)));
+ i6 += 16;
+
+ const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
+ const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
+
+ const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
+ const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 224)));
+ i7 += 16;
+
+ const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
+ const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
+
+ const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
+ const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 256)));
+ i8 += 16;
+
+ const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
+ const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
+
+ const __m256i vi9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i9));
+ const __m256i vk9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 288)));
+ i9 += 16;
+
+ const __m256i vprod9x0123456789ABCDEF = _mm256_mullo_epi16(vi9x0123456789ABCDEF, vk9x0123456789ABCDEF);
+ const __m128i vprod9x89ABCDEF = _mm256_extracti128_si256(vprod9x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod9x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod9x89ABCDEF));
+
+ const __m256i vi10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i10));
+ const __m256i vk10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 320)));
+ i10 += 16;
+
+ const __m256i vprod10x0123456789ABCDEF = _mm256_mullo_epi16(vi10x0123456789ABCDEF, vk10x0123456789ABCDEF);
+ const __m128i vprod10x89ABCDEF = _mm256_extracti128_si256(vprod10x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod10x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod10x89ABCDEF));
+
+ const __m256i vi11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i11));
+ const __m256i vk11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 352)));
+ i11 += 16;
+
+ const __m256i vprod11x0123456789ABCDEF = _mm256_mullo_epi16(vi11x0123456789ABCDEF, vk11x0123456789ABCDEF);
+ const __m128i vprod11x89ABCDEF = _mm256_extracti128_si256(vprod11x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod11x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod11x89ABCDEF));
+
+ const __m256i vi12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i12));
+ const __m256i vk12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 384)));
+ i12 += 16;
+
+ const __m256i vprod12x0123456789ABCDEF = _mm256_mullo_epi16(vi12x0123456789ABCDEF, vk12x0123456789ABCDEF);
+ const __m128i vprod12x89ABCDEF = _mm256_extracti128_si256(vprod12x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod12x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod12x89ABCDEF));
+
+ const __m256i vi13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i13));
+ const __m256i vk13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 416)));
+ i13 += 16;
+
+ const __m256i vprod13x0123456789ABCDEF = _mm256_mullo_epi16(vi13x0123456789ABCDEF, vk13x0123456789ABCDEF);
+ const __m128i vprod13x89ABCDEF = _mm256_extracti128_si256(vprod13x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod13x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod13x89ABCDEF));
+
+ const __m256i vi14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i14));
+ const __m256i vk14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 448)));
+ i14 += 16;
+
+ const __m256i vprod14x0123456789ABCDEF = _mm256_mullo_epi16(vi14x0123456789ABCDEF, vk14x0123456789ABCDEF);
+ const __m128i vprod14x89ABCDEF = _mm256_extracti128_si256(vprod14x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod14x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod14x89ABCDEF));
+
+ const __m256i vi15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i15));
+ const __m256i vk15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 480)));
+ i15 += 16;
+
+ const __m256i vprod15x0123456789ABCDEF = _mm256_mullo_epi16(vi15x0123456789ABCDEF, vk15x0123456789ABCDEF);
+ const __m128i vprod15x89ABCDEF = _mm256_extracti128_si256(vprod15x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod15x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod15x89ABCDEF));
+
+ const __m256i vi16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i16));
+ const __m256i vk16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 512)));
+ i16 += 16;
+
+ const __m256i vprod16x0123456789ABCDEF = _mm256_mullo_epi16(vi16x0123456789ABCDEF, vk16x0123456789ABCDEF);
+ const __m128i vprod16x89ABCDEF = _mm256_extracti128_si256(vprod16x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod16x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod16x89ABCDEF));
+
+ const __m256i vi17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i17));
+ const __m256i vk17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 544)));
+ i17 += 16;
+
+ const __m256i vprod17x0123456789ABCDEF = _mm256_mullo_epi16(vi17x0123456789ABCDEF, vk17x0123456789ABCDEF);
+ const __m128i vprod17x89ABCDEF = _mm256_extracti128_si256(vprod17x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod17x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod17x89ABCDEF));
+
+ const __m256i vi18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i18));
+ const __m256i vk18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 576)));
+ i18 += 16;
+
+ const __m256i vprod18x0123456789ABCDEF = _mm256_mullo_epi16(vi18x0123456789ABCDEF, vk18x0123456789ABCDEF);
+ const __m128i vprod18x89ABCDEF = _mm256_extracti128_si256(vprod18x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod18x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod18x89ABCDEF));
+
+ const __m256i vi19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i19));
+ const __m256i vk19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 608)));
+ i19 += 16;
+
+ const __m256i vprod19x0123456789ABCDEF = _mm256_mullo_epi16(vi19x0123456789ABCDEF, vk19x0123456789ABCDEF);
+ const __m128i vprod19x89ABCDEF = _mm256_extracti128_si256(vprod19x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod19x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod19x89ABCDEF));
+
+ const __m256i vi20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i20));
+ const __m256i vk20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 640)));
+ i20 += 16;
+
+ const __m256i vprod20x0123456789ABCDEF = _mm256_mullo_epi16(vi20x0123456789ABCDEF, vk20x0123456789ABCDEF);
+ const __m128i vprod20x89ABCDEF = _mm256_extracti128_si256(vprod20x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod20x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod20x89ABCDEF));
+
+ const __m256i vi21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i21));
+ const __m256i vk21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 672)));
+ i21 += 16;
+
+ const __m256i vprod21x0123456789ABCDEF = _mm256_mullo_epi16(vi21x0123456789ABCDEF, vk21x0123456789ABCDEF);
+ const __m128i vprod21x89ABCDEF = _mm256_extracti128_si256(vprod21x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod21x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod21x89ABCDEF));
+
+ const __m256i vi22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i22));
+ const __m256i vk22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 704)));
+ i22 += 16;
+
+ const __m256i vprod22x0123456789ABCDEF = _mm256_mullo_epi16(vi22x0123456789ABCDEF, vk22x0123456789ABCDEF);
+ const __m128i vprod22x89ABCDEF = _mm256_extracti128_si256(vprod22x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod22x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod22x89ABCDEF));
+
+ const __m256i vi23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i23));
+ const __m256i vk23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 736)));
+ i23 += 16;
+
+ const __m256i vprod23x0123456789ABCDEF = _mm256_mullo_epi16(vi23x0123456789ABCDEF, vk23x0123456789ABCDEF);
+ const __m128i vprod23x89ABCDEF = _mm256_extracti128_si256(vprod23x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod23x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod23x89ABCDEF));
+
+ const __m256i vi24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i24));
+ const __m256i vk24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 768)));
+ i24 += 16;
+
+ const __m256i vprod24x0123456789ABCDEF = _mm256_mullo_epi16(vi24x0123456789ABCDEF, vk24x0123456789ABCDEF);
+ const __m128i vprod24x89ABCDEF = _mm256_extracti128_si256(vprod24x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod24x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
+
+ w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ k += 16;
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+ __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+
+ __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
+
+ if XNN_LIKELY(c >= 16) {
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ output += 16;
+ c -= 16;
+ } else {
+ if (c & 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
+ vout0123456789ABCDEF = _mm_unpackhi_epi64(vout0123456789ABCDEF, vout0123456789ABCDEF);
+ output += 8;
+ }
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456789ABCDEF);
+ vout0123456789ABCDEF = _mm_srli_epi64(vout0123456789ABCDEF, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456789ABCDEF, 0);
+ vout0123456789ABCDEF = _mm_srli_epi32(vout0123456789ABCDEF, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456789ABCDEF, 0);
+ output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-fp32-avx2-mul32.c b/src/qs8-dwconv/gen/up32x25-minmax-fp32-avx2-mul32.c
new file mode 100644
index 0000000..f8017a2
--- /dev/null
+++ b/src/qs8-dwconv/gen/up32x25-minmax-fp32-avx2-mul32.c
@@ -0,0 +1,778 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul32.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up32x25__avx2_mul32(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 32; c -= 32) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+ __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
+ __m256i vaccOPQRSTUV = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 24 * sizeof(int32_t)));
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
+ const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+ const __m256i vi0xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 16)));
+ const __m256i vk0xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ const __m256i vi0xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 24)));
+ const __m256i vk0xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+ i0 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi0xGHIJKLMN, vk0xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi0xOPQRSTUV, vk0xOPQRSTUV));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
+ const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+ const __m256i vi1xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 16)));
+ const __m256i vk1xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ const __m256i vi1xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 24)));
+ const __m256i vk1xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+ i1 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi1xGHIJKLMN, vk1xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi1xOPQRSTUV, vk1xOPQRSTUV));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
+ const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 72 * sizeof(int8_t))));
+ const __m256i vi2xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 16)));
+ const __m256i vk2xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ const __m256i vi2xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 24)));
+ const __m256i vk2xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 88 * sizeof(int8_t))));
+ i2 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi2xGHIJKLMN, vk2xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi2xOPQRSTUV, vk2xOPQRSTUV));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
+ const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 104 * sizeof(int8_t))));
+ const __m256i vi3xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 16)));
+ const __m256i vk3xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ const __m256i vi3xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 24)));
+ const __m256i vk3xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 120 * sizeof(int8_t))));
+ i3 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi3xGHIJKLMN, vk3xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi3xOPQRSTUV, vk3xOPQRSTUV));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
+ const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 136 * sizeof(int8_t))));
+ const __m256i vi4xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 16)));
+ const __m256i vk4xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ const __m256i vi4xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 24)));
+ const __m256i vk4xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 152 * sizeof(int8_t))));
+ i4 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi4xGHIJKLMN, vk4xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi4xOPQRSTUV, vk4xOPQRSTUV));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
+ const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 168 * sizeof(int8_t))));
+ const __m256i vi5xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 16)));
+ const __m256i vk5xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ const __m256i vi5xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 24)));
+ const __m256i vk5xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 184 * sizeof(int8_t))));
+ i5 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi5xGHIJKLMN, vk5xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi5xOPQRSTUV, vk5xOPQRSTUV));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
+ const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 200 * sizeof(int8_t))));
+ const __m256i vi6xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 16)));
+ const __m256i vk6xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+ const __m256i vi6xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 24)));
+ const __m256i vk6xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 216 * sizeof(int8_t))));
+ i6 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi6xGHIJKLMN, vk6xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi6xOPQRSTUV, vk6xOPQRSTUV));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 224 * sizeof(int8_t))));
+ const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
+ const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 232 * sizeof(int8_t))));
+ const __m256i vi7xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 16)));
+ const __m256i vk7xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 240 * sizeof(int8_t))));
+ const __m256i vi7xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 24)));
+ const __m256i vk7xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 248 * sizeof(int8_t))));
+ i7 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi7xGHIJKLMN, vk7xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi7xOPQRSTUV, vk7xOPQRSTUV));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 256 * sizeof(int8_t))));
+ const __m256i vi8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 8)));
+ const __m256i vk8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 264 * sizeof(int8_t))));
+ const __m256i vi8xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 16)));
+ const __m256i vk8xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 272 * sizeof(int8_t))));
+ const __m256i vi8xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 24)));
+ const __m256i vk8xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 280 * sizeof(int8_t))));
+ i8 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi8x89ABCDEF, vk8x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi8xGHIJKLMN, vk8xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi8xOPQRSTUV, vk8xOPQRSTUV));
+
+ const __m256i vi9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i9));
+ const __m256i vk9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t))));
+ const __m256i vi9x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i9 + 8)));
+ const __m256i vk9x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 296 * sizeof(int8_t))));
+ const __m256i vi9xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i9 + 16)));
+ const __m256i vk9xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 304 * sizeof(int8_t))));
+ const __m256i vi9xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i9 + 24)));
+ const __m256i vk9xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 312 * sizeof(int8_t))));
+ i9 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi9x01234567, vk9x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi9x89ABCDEF, vk9x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi9xGHIJKLMN, vk9xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi9xOPQRSTUV, vk9xOPQRSTUV));
+
+ const __m256i vi10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i10));
+ const __m256i vk10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 320 * sizeof(int8_t))));
+ const __m256i vi10x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i10 + 8)));
+ const __m256i vk10x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 328 * sizeof(int8_t))));
+ const __m256i vi10xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i10 + 16)));
+ const __m256i vk10xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 336 * sizeof(int8_t))));
+ const __m256i vi10xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i10 + 24)));
+ const __m256i vk10xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 344 * sizeof(int8_t))));
+ i10 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi10x01234567, vk10x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi10x89ABCDEF, vk10x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi10xGHIJKLMN, vk10xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi10xOPQRSTUV, vk10xOPQRSTUV));
+
+ const __m256i vi11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i11));
+ const __m256i vk11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 352 * sizeof(int8_t))));
+ const __m256i vi11x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i11 + 8)));
+ const __m256i vk11x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 360 * sizeof(int8_t))));
+ const __m256i vi11xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i11 + 16)));
+ const __m256i vk11xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 368 * sizeof(int8_t))));
+ const __m256i vi11xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i11 + 24)));
+ const __m256i vk11xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 376 * sizeof(int8_t))));
+ i11 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi11x01234567, vk11x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi11x89ABCDEF, vk11x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi11xGHIJKLMN, vk11xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi11xOPQRSTUV, vk11xOPQRSTUV));
+
+ const __m256i vi12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i12));
+ const __m256i vk12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 384 * sizeof(int8_t))));
+ const __m256i vi12x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i12 + 8)));
+ const __m256i vk12x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 392 * sizeof(int8_t))));
+ const __m256i vi12xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i12 + 16)));
+ const __m256i vk12xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 400 * sizeof(int8_t))));
+ const __m256i vi12xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i12 + 24)));
+ const __m256i vk12xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 408 * sizeof(int8_t))));
+ i12 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi12x01234567, vk12x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi12x89ABCDEF, vk12x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi12xGHIJKLMN, vk12xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi12xOPQRSTUV, vk12xOPQRSTUV));
+
+ const __m256i vi13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i13));
+ const __m256i vk13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 416 * sizeof(int8_t))));
+ const __m256i vi13x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i13 + 8)));
+ const __m256i vk13x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 424 * sizeof(int8_t))));
+ const __m256i vi13xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i13 + 16)));
+ const __m256i vk13xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 432 * sizeof(int8_t))));
+ const __m256i vi13xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i13 + 24)));
+ const __m256i vk13xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 440 * sizeof(int8_t))));
+ i13 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi13x01234567, vk13x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi13x89ABCDEF, vk13x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi13xGHIJKLMN, vk13xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi13xOPQRSTUV, vk13xOPQRSTUV));
+
+ const __m256i vi14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i14));
+ const __m256i vk14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 448 * sizeof(int8_t))));
+ const __m256i vi14x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i14 + 8)));
+ const __m256i vk14x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 456 * sizeof(int8_t))));
+ const __m256i vi14xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i14 + 16)));
+ const __m256i vk14xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 464 * sizeof(int8_t))));
+ const __m256i vi14xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i14 + 24)));
+ const __m256i vk14xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 472 * sizeof(int8_t))));
+ i14 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi14x01234567, vk14x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi14x89ABCDEF, vk14x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi14xGHIJKLMN, vk14xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi14xOPQRSTUV, vk14xOPQRSTUV));
+
+ const __m256i vi15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i15));
+ const __m256i vk15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 480 * sizeof(int8_t))));
+ const __m256i vi15x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i15 + 8)));
+ const __m256i vk15x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 488 * sizeof(int8_t))));
+ const __m256i vi15xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i15 + 16)));
+ const __m256i vk15xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 496 * sizeof(int8_t))));
+ const __m256i vi15xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i15 + 24)));
+ const __m256i vk15xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 504 * sizeof(int8_t))));
+ i15 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi15x01234567, vk15x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi15x89ABCDEF, vk15x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi15xGHIJKLMN, vk15xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi15xOPQRSTUV, vk15xOPQRSTUV));
+
+ const __m256i vi16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i16));
+ const __m256i vk16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 512 * sizeof(int8_t))));
+ const __m256i vi16x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i16 + 8)));
+ const __m256i vk16x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 520 * sizeof(int8_t))));
+ const __m256i vi16xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i16 + 16)));
+ const __m256i vk16xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 528 * sizeof(int8_t))));
+ const __m256i vi16xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i16 + 24)));
+ const __m256i vk16xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 536 * sizeof(int8_t))));
+ i16 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi16x01234567, vk16x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi16x89ABCDEF, vk16x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi16xGHIJKLMN, vk16xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi16xOPQRSTUV, vk16xOPQRSTUV));
+
+ const __m256i vi17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i17));
+ const __m256i vk17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 544 * sizeof(int8_t))));
+ const __m256i vi17x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i17 + 8)));
+ const __m256i vk17x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 552 * sizeof(int8_t))));
+ const __m256i vi17xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i17 + 16)));
+ const __m256i vk17xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 560 * sizeof(int8_t))));
+ const __m256i vi17xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i17 + 24)));
+ const __m256i vk17xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 568 * sizeof(int8_t))));
+ i17 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi17x01234567, vk17x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi17x89ABCDEF, vk17x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi17xGHIJKLMN, vk17xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi17xOPQRSTUV, vk17xOPQRSTUV));
+
+ const __m256i vi18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i18));
+ const __m256i vk18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 576 * sizeof(int8_t))));
+ const __m256i vi18x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i18 + 8)));
+ const __m256i vk18x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 584 * sizeof(int8_t))));
+ const __m256i vi18xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i18 + 16)));
+ const __m256i vk18xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 592 * sizeof(int8_t))));
+ const __m256i vi18xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i18 + 24)));
+ const __m256i vk18xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 600 * sizeof(int8_t))));
+ i18 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi18x01234567, vk18x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi18x89ABCDEF, vk18x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi18xGHIJKLMN, vk18xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi18xOPQRSTUV, vk18xOPQRSTUV));
+
+ const __m256i vi19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i19));
+ const __m256i vk19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 608 * sizeof(int8_t))));
+ const __m256i vi19x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i19 + 8)));
+ const __m256i vk19x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 616 * sizeof(int8_t))));
+ const __m256i vi19xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i19 + 16)));
+ const __m256i vk19xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 624 * sizeof(int8_t))));
+ const __m256i vi19xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i19 + 24)));
+ const __m256i vk19xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 632 * sizeof(int8_t))));
+ i19 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi19x01234567, vk19x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi19x89ABCDEF, vk19x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi19xGHIJKLMN, vk19xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi19xOPQRSTUV, vk19xOPQRSTUV));
+
+ const __m256i vi20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i20));
+ const __m256i vk20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 640 * sizeof(int8_t))));
+ const __m256i vi20x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i20 + 8)));
+ const __m256i vk20x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 648 * sizeof(int8_t))));
+ const __m256i vi20xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i20 + 16)));
+ const __m256i vk20xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 656 * sizeof(int8_t))));
+ const __m256i vi20xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i20 + 24)));
+ const __m256i vk20xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 664 * sizeof(int8_t))));
+ i20 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi20x01234567, vk20x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi20x89ABCDEF, vk20x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi20xGHIJKLMN, vk20xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi20xOPQRSTUV, vk20xOPQRSTUV));
+
+ const __m256i vi21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i21));
+ const __m256i vk21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 672 * sizeof(int8_t))));
+ const __m256i vi21x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i21 + 8)));
+ const __m256i vk21x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 680 * sizeof(int8_t))));
+ const __m256i vi21xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i21 + 16)));
+ const __m256i vk21xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 688 * sizeof(int8_t))));
+ const __m256i vi21xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i21 + 24)));
+ const __m256i vk21xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 696 * sizeof(int8_t))));
+ i21 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi21x01234567, vk21x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi21x89ABCDEF, vk21x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi21xGHIJKLMN, vk21xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi21xOPQRSTUV, vk21xOPQRSTUV));
+
+ const __m256i vi22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i22));
+ const __m256i vk22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 704 * sizeof(int8_t))));
+ const __m256i vi22x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i22 + 8)));
+ const __m256i vk22x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 712 * sizeof(int8_t))));
+ const __m256i vi22xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i22 + 16)));
+ const __m256i vk22xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 720 * sizeof(int8_t))));
+ const __m256i vi22xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i22 + 24)));
+ const __m256i vk22xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 728 * sizeof(int8_t))));
+ i22 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi22x01234567, vk22x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi22x89ABCDEF, vk22x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi22xGHIJKLMN, vk22xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi22xOPQRSTUV, vk22xOPQRSTUV));
+
+ const __m256i vi23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i23));
+ const __m256i vk23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 736 * sizeof(int8_t))));
+ const __m256i vi23x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i23 + 8)));
+ const __m256i vk23x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 744 * sizeof(int8_t))));
+ const __m256i vi23xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i23 + 16)));
+ const __m256i vk23xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 752 * sizeof(int8_t))));
+ const __m256i vi23xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i23 + 24)));
+ const __m256i vk23xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 760 * sizeof(int8_t))));
+ i23 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi23x01234567, vk23x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi23x89ABCDEF, vk23x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi23xGHIJKLMN, vk23xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi23xOPQRSTUV, vk23xOPQRSTUV));
+
+ const __m256i vi24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i24));
+ const __m256i vk24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 768 * sizeof(int8_t))));
+ const __m256i vi24x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i24 + 8)));
+ const __m256i vk24x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 776 * sizeof(int8_t))));
+ const __m256i vi24xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i24 + 16)));
+ const __m256i vk24xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 784 * sizeof(int8_t))));
+ const __m256i vi24xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i24 + 24)));
+ const __m256i vk24xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 792 * sizeof(int8_t))));
+ i24 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi24x89ABCDEF, vk24x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi24xGHIJKLMN, vk24xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi24xOPQRSTUV, vk24xOPQRSTUV));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 800 * sizeof(int8_t));
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+ __m256 vscaledGHIJKLMN = _mm256_cvtepi32_ps(vaccGHIJKLMN);
+ __m256 vscaledOPQRSTUV = _mm256_cvtepi32_ps(vaccOPQRSTUV);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+ vscaledGHIJKLMN = _mm256_mul_ps(vscaledGHIJKLMN, vscale);
+ vscaledOPQRSTUV = _mm256_mul_ps(vscaledOPQRSTUV, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+ vaccGHIJKLMN = _mm256_cvtps_epi32(vscaledGHIJKLMN);
+ vaccOPQRSTUV = _mm256_cvtps_epi32(vscaledOPQRSTUV);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+ __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ __m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
+ voutGHIJKLMNOPQRSTUV = _mm_min_epi8(voutGHIJKLMNOPQRSTUV, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ _mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
+ output += 32;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((uintptr_t) w + 32 * sizeof(int32_t));
+ do {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) k));
+ i0 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 32)));
+ i1 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 64)));
+ i2 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 96)));
+ i3 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 128)));
+ i4 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 160)));
+ i5 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 192)));
+ i6 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 224)));
+ i7 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 256)));
+ i8 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ const __m256i vi9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i9));
+ const __m256i vk9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 288)));
+ i9 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi9x01234567, vk9x01234567));
+
+ const __m256i vi10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i10));
+ const __m256i vk10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 320)));
+ i10 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi10x01234567, vk10x01234567));
+
+ const __m256i vi11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i11));
+ const __m256i vk11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 352)));
+ i11 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi11x01234567, vk11x01234567));
+
+ const __m256i vi12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i12));
+ const __m256i vk12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 384)));
+ i12 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi12x01234567, vk12x01234567));
+
+ const __m256i vi13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i13));
+ const __m256i vk13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 416)));
+ i13 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi13x01234567, vk13x01234567));
+
+ const __m256i vi14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i14));
+ const __m256i vk14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 448)));
+ i14 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi14x01234567, vk14x01234567));
+
+ const __m256i vi15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i15));
+ const __m256i vk15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 480)));
+ i15 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi15x01234567, vk15x01234567));
+
+ const __m256i vi16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i16));
+ const __m256i vk16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 512)));
+ i16 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi16x01234567, vk16x01234567));
+
+ const __m256i vi17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i17));
+ const __m256i vk17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 544)));
+ i17 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi17x01234567, vk17x01234567));
+
+ const __m256i vi18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i18));
+ const __m256i vk18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 576)));
+ i18 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi18x01234567, vk18x01234567));
+
+ const __m256i vi19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i19));
+ const __m256i vk19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 608)));
+ i19 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi19x01234567, vk19x01234567));
+
+ const __m256i vi20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i20));
+ const __m256i vk20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 640)));
+ i20 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi20x01234567, vk20x01234567));
+
+ const __m256i vi21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i21));
+ const __m256i vk21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 672)));
+ i21 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi21x01234567, vk21x01234567));
+
+ const __m256i vi22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i22));
+ const __m256i vk22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 704)));
+ i22 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi22x01234567, vk22x01234567));
+
+ const __m256i vi23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i23));
+ const __m256i vk23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 736)));
+ i23 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi23x01234567, vk23x01234567));
+
+ const __m256i vi24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i24));
+ const __m256i vk24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 768)));
+ i24 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
+
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ k += 8;
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ if XNN_LIKELY(c >= 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456701234567);
+ output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
+ vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456701234567, 0);
+ vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
+ output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx2-mul16.c b/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx2-mul16.c
index e984902..7965487 100644
--- a/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx2-mul16.c
+++ b/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx2-mul16.c
@@ -542,9 +542,8 @@
vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod24xOPQRSTUV));
w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 800 * sizeof(int8_t));
-
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -574,7 +573,7 @@
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
const __m256i vq31prodOPQRSTUV = _mm256_blend_epi16(vq31prodOQSU, vq31prodPRTV, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -584,8 +583,8 @@
const __m256i vremOPQRSTUV =
_mm256_add_epi32(_mm256_and_si256(vq31prodOPQRSTUV, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodOPQRSTUV));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -595,15 +594,15 @@
vaccOPQRSTUV =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodOPQRSTUV, vshift), _mm256_cmpgt_epi32(vremOPQRSTUV, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
const __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
@@ -848,8 +847,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
k += 16;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -867,25 +866,25 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx2-mul32.c b/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx2-mul32.c
index 109cb1c..3d51fd6 100644
--- a/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx2-mul32.c
@@ -543,8 +543,8 @@
w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 800 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -574,7 +574,7 @@
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
const __m256i vq31prodOPQRSTUV = _mm256_blend_epi16(vq31prodOQSU, vq31prodPRTV, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -584,8 +584,8 @@
const __m256i vremOPQRSTUV =
_mm256_add_epi32(_mm256_and_si256(vq31prodOPQRSTUV, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodOPQRSTUV));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -595,15 +595,15 @@
vaccOPQRSTUV =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodOPQRSTUV, vshift), _mm256_cmpgt_epi32(vremOPQRSTUV, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
@@ -772,8 +772,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -785,21 +785,21 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx512skx-mul32.c b/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx512skx-mul32.c
index 4d0bcad..3dd25d0 100644
--- a/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx512skx-mul32.c
+++ b/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-avx512skx-mul32.c
@@ -31,14 +31,14 @@
assert(output_width != 0);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 5, 1, 6, 2, 4, 0);
do {
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-neon-mul16.c b/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-neon-mul16.c
index d623520..2ae31b7 100644
--- a/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up32x25-minmax-gemmlowp-neon-mul16.c
@@ -29,12 +29,12 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-fp32-avx2-mul16.c b/src/qs8-dwconv/gen/up32x9-minmax-fp32-avx2-mul16.c
new file mode 100644
index 0000000..258a80b
--- /dev/null
+++ b/src/qs8-dwconv/gen/up32x9-minmax-fp32-avx2-mul16.c
@@ -0,0 +1,401 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul16.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up32x9__avx2_mul16(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 32; c -= 32) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+ __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
+ __m256i vaccOPQRSTUV = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 24 * sizeof(int32_t)));
+
+
+ const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
+ const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ const __m256i vi0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i0 + 16)));
+ const __m256i vk0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ i0 += 32;
+
+ const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
+ const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
+ const __m256i vprod0xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi0xGHIJKLMNOPQRSTUV, vk0xGHIJKLMNOPQRSTUV);
+ const __m128i vprod0xOPQRSTUV = _mm256_extracti128_si256(vprod0xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod0xOPQRSTUV));
+
+ const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
+ const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ const __m256i vi1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i1 + 16)));
+ const __m256i vk1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ i1 += 32;
+
+ const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
+ const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
+ const __m256i vprod1xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi1xGHIJKLMNOPQRSTUV, vk1xGHIJKLMNOPQRSTUV);
+ const __m128i vprod1xOPQRSTUV = _mm256_extracti128_si256(vprod1xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod1xOPQRSTUV));
+
+ const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
+ const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ const __m256i vi2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i2 + 16)));
+ const __m256i vk2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ i2 += 32;
+
+ const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
+ const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
+ const __m256i vprod2xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi2xGHIJKLMNOPQRSTUV, vk2xGHIJKLMNOPQRSTUV);
+ const __m128i vprod2xOPQRSTUV = _mm256_extracti128_si256(vprod2xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod2xOPQRSTUV));
+
+ const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
+ const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ const __m256i vi3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i3 + 16)));
+ const __m256i vk3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ i3 += 32;
+
+ const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
+ const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
+ const __m256i vprod3xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi3xGHIJKLMNOPQRSTUV, vk3xGHIJKLMNOPQRSTUV);
+ const __m128i vprod3xOPQRSTUV = _mm256_extracti128_si256(vprod3xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod3xOPQRSTUV));
+
+ const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
+ const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ const __m256i vi4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i4 + 16)));
+ const __m256i vk4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ i4 += 32;
+
+ const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
+ const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
+ const __m256i vprod4xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi4xGHIJKLMNOPQRSTUV, vk4xGHIJKLMNOPQRSTUV);
+ const __m128i vprod4xOPQRSTUV = _mm256_extracti128_si256(vprod4xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod4xOPQRSTUV));
+
+ const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
+ const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ const __m256i vi5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i5 + 16)));
+ const __m256i vk5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ i5 += 32;
+
+ const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
+ const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
+ const __m256i vprod5xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi5xGHIJKLMNOPQRSTUV, vk5xGHIJKLMNOPQRSTUV);
+ const __m128i vprod5xOPQRSTUV = _mm256_extracti128_si256(vprod5xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod5xOPQRSTUV));
+
+ const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
+ const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ const __m256i vi6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i6 + 16)));
+ const __m256i vk6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+ i6 += 32;
+
+ const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
+ const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
+ const __m256i vprod6xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi6xGHIJKLMNOPQRSTUV, vk6xGHIJKLMNOPQRSTUV);
+ const __m128i vprod6xOPQRSTUV = _mm256_extracti128_si256(vprod6xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod6xOPQRSTUV));
+
+ const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
+ const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 224 * sizeof(int8_t))));
+ const __m256i vi7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i7 + 16)));
+ const __m256i vk7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 240 * sizeof(int8_t))));
+ i7 += 32;
+
+ const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
+ const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
+ const __m256i vprod7xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi7xGHIJKLMNOPQRSTUV, vk7xGHIJKLMNOPQRSTUV);
+ const __m128i vprod7xOPQRSTUV = _mm256_extracti128_si256(vprod7xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod7xOPQRSTUV));
+
+ const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
+ const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 256 * sizeof(int8_t))));
+ const __m256i vi8xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i8 + 16)));
+ const __m256i vk8xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 272 * sizeof(int8_t))));
+ i8 += 32;
+
+ const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
+ const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
+ const __m256i vprod8xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi8xGHIJKLMNOPQRSTUV, vk8xGHIJKLMNOPQRSTUV);
+ const __m128i vprod8xOPQRSTUV = _mm256_extracti128_si256(vprod8xGHIJKLMNOPQRSTUV, 1);
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8xGHIJKLMNOPQRSTUV)));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod8xOPQRSTUV));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t));
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+ __m256 vscaledGHIJKLMN = _mm256_cvtepi32_ps(vaccGHIJKLMN);
+ __m256 vscaledOPQRSTUV = _mm256_cvtepi32_ps(vaccOPQRSTUV);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+ vscaledGHIJKLMN = _mm256_mul_ps(vscaledGHIJKLMN, vscale);
+ vscaledOPQRSTUV = _mm256_mul_ps(vscaledOPQRSTUV, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+ vaccGHIJKLMN = _mm256_cvtps_epi32(vscaledGHIJKLMN);
+ vaccOPQRSTUV = _mm256_cvtps_epi32(vscaledOPQRSTUV);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+ const __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ __m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
+ voutGHIJKLMNOPQRSTUV = _mm_min_epi8(voutGHIJKLMNOPQRSTUV, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ _mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
+ output += 32;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((uintptr_t) w + 32 * sizeof(int32_t));
+ do {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+
+
+ const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
+ const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) k));
+ i0 += 16;
+
+ const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
+ const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
+
+ const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
+ const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 32)));
+ i1 += 16;
+
+ const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
+ const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
+
+ const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
+ const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 64)));
+ i2 += 16;
+
+ const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
+ const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
+
+ const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
+ const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 96)));
+ i3 += 16;
+
+ const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
+ const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
+
+ const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
+ const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 128)));
+ i4 += 16;
+
+ const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
+ const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
+
+ const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
+ const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 160)));
+ i5 += 16;
+
+ const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
+ const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
+
+ const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
+ const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 192)));
+ i6 += 16;
+
+ const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
+ const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
+
+ const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
+ const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 224)));
+ i7 += 16;
+
+ const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
+ const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
+
+ const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
+ const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 256)));
+ i8 += 16;
+
+ const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
+ const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
+
+ w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ k += 16;
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+ __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+
+ __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
+
+ if XNN_LIKELY(c >= 16) {
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ output += 16;
+ c -= 16;
+ } else {
+ if (c & 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
+ vout0123456789ABCDEF = _mm_unpackhi_epi64(vout0123456789ABCDEF, vout0123456789ABCDEF);
+ output += 8;
+ }
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456789ABCDEF);
+ vout0123456789ABCDEF = _mm_srli_epi64(vout0123456789ABCDEF, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456789ABCDEF, 0);
+ vout0123456789ABCDEF = _mm_srli_epi32(vout0123456789ABCDEF, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456789ABCDEF, 0);
+ output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-fp32-avx2-mul32.c b/src/qs8-dwconv/gen/up32x9-minmax-fp32-avx2-mul32.c
new file mode 100644
index 0000000..5bde3a6
--- /dev/null
+++ b/src/qs8-dwconv/gen/up32x9-minmax-fp32-avx2-mul32.c
@@ -0,0 +1,362 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul32.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up32x9__avx2_mul32(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 32; c -= 32) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+ __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
+ __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
+ __m256i vaccOPQRSTUV = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 24 * sizeof(int32_t)));
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
+ const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+ const __m256i vi0xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 16)));
+ const __m256i vk0xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ const __m256i vi0xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 24)));
+ const __m256i vk0xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+ i0 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi0xGHIJKLMN, vk0xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi0xOPQRSTUV, vk0xOPQRSTUV));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
+ const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+ const __m256i vi1xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 16)));
+ const __m256i vk1xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ const __m256i vi1xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 24)));
+ const __m256i vk1xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+ i1 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi1xGHIJKLMN, vk1xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi1xOPQRSTUV, vk1xOPQRSTUV));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
+ const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 72 * sizeof(int8_t))));
+ const __m256i vi2xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 16)));
+ const __m256i vk2xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ const __m256i vi2xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 24)));
+ const __m256i vk2xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 88 * sizeof(int8_t))));
+ i2 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi2xGHIJKLMN, vk2xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi2xOPQRSTUV, vk2xOPQRSTUV));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
+ const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 104 * sizeof(int8_t))));
+ const __m256i vi3xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 16)));
+ const __m256i vk3xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ const __m256i vi3xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 24)));
+ const __m256i vk3xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 120 * sizeof(int8_t))));
+ i3 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi3xGHIJKLMN, vk3xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi3xOPQRSTUV, vk3xOPQRSTUV));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
+ const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 136 * sizeof(int8_t))));
+ const __m256i vi4xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 16)));
+ const __m256i vk4xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ const __m256i vi4xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 24)));
+ const __m256i vk4xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 152 * sizeof(int8_t))));
+ i4 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi4xGHIJKLMN, vk4xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi4xOPQRSTUV, vk4xOPQRSTUV));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
+ const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 168 * sizeof(int8_t))));
+ const __m256i vi5xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 16)));
+ const __m256i vk5xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ const __m256i vi5xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 24)));
+ const __m256i vk5xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 184 * sizeof(int8_t))));
+ i5 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi5xGHIJKLMN, vk5xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi5xOPQRSTUV, vk5xOPQRSTUV));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
+ const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 200 * sizeof(int8_t))));
+ const __m256i vi6xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 16)));
+ const __m256i vk6xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 208 * sizeof(int8_t))));
+ const __m256i vi6xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 24)));
+ const __m256i vk6xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 216 * sizeof(int8_t))));
+ i6 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi6xGHIJKLMN, vk6xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi6xOPQRSTUV, vk6xOPQRSTUV));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 224 * sizeof(int8_t))));
+ const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
+ const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 232 * sizeof(int8_t))));
+ const __m256i vi7xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 16)));
+ const __m256i vk7xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 240 * sizeof(int8_t))));
+ const __m256i vi7xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 24)));
+ const __m256i vk7xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 248 * sizeof(int8_t))));
+ i7 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi7xGHIJKLMN, vk7xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi7xOPQRSTUV, vk7xOPQRSTUV));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 256 * sizeof(int8_t))));
+ const __m256i vi8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 8)));
+ const __m256i vk8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 264 * sizeof(int8_t))));
+ const __m256i vi8xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 16)));
+ const __m256i vk8xGHIJKLMN = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 272 * sizeof(int8_t))));
+ const __m256i vi8xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 24)));
+ const __m256i vk8xOPQRSTUV = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 280 * sizeof(int8_t))));
+ i8 += 32;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+ vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi8x89ABCDEF, vk8x89ABCDEF));
+ vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_mullo_epi32(vi8xGHIJKLMN, vk8xGHIJKLMN));
+ vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_mullo_epi32(vi8xOPQRSTUV, vk8xOPQRSTUV));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t));
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
+ __m256 vscaledGHIJKLMN = _mm256_cvtepi32_ps(vaccGHIJKLMN);
+ __m256 vscaledOPQRSTUV = _mm256_cvtepi32_ps(vaccOPQRSTUV);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+ vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
+ vscaledGHIJKLMN = _mm256_mul_ps(vscaledGHIJKLMN, vscale);
+ vscaledOPQRSTUV = _mm256_mul_ps(vscaledOPQRSTUV, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+ vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
+ vaccGHIJKLMN = _mm256_cvtps_epi32(vscaledGHIJKLMN);
+ vaccOPQRSTUV = _mm256_cvtps_epi32(vscaledOPQRSTUV);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+ __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
+
+ __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ __m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
+ voutGHIJKLMNOPQRSTUV = _mm_min_epi8(voutGHIJKLMNOPQRSTUV, voutput_max);
+
+ _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
+ _mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
+ output += 32;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((uintptr_t) w + 32 * sizeof(int32_t));
+ do {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) k));
+ i0 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 32)));
+ i1 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 64)));
+ i2 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 96)));
+ i3 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 128)));
+ i4 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 160)));
+ i5 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 192)));
+ i6 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 224)));
+ i7 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (k + 256)));
+ i8 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ k += 8;
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ if XNN_LIKELY(c >= 8) {
+ _mm_storel_epi64((__m128i*) output, vout0123456701234567);
+ output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
+ vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456701234567, 0);
+ vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
+ output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx2-mul16.c b/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx2-mul16.c
index 6dfc4f9..853b119 100644
--- a/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx2-mul16.c
+++ b/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx2-mul16.c
@@ -222,9 +222,8 @@
vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod8xOPQRSTUV));
w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t));
-
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -254,7 +253,7 @@
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
const __m256i vq31prodOPQRSTUV = _mm256_blend_epi16(vq31prodOQSU, vq31prodPRTV, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -264,8 +263,8 @@
const __m256i vremOPQRSTUV =
_mm256_add_epi32(_mm256_and_si256(vq31prodOPQRSTUV, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodOPQRSTUV));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -275,15 +274,15 @@
vaccOPQRSTUV =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodOPQRSTUV, vshift), _mm256_cmpgt_epi32(vremOPQRSTUV, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
const __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
@@ -384,8 +383,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
k += 16;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -403,25 +402,25 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx2-mul32.c b/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx2-mul32.c
index 8e3923a..cc5dcf2 100644
--- a/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx2-mul32.c
@@ -223,8 +223,8 @@
w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -254,7 +254,7 @@
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
const __m256i vq31prodOPQRSTUV = _mm256_blend_epi16(vq31prodOQSU, vq31prodPRTV, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -264,8 +264,8 @@
const __m256i vremOPQRSTUV =
_mm256_add_epi32(_mm256_and_si256(vq31prodOPQRSTUV, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodOPQRSTUV));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -275,15 +275,15 @@
vaccOPQRSTUV =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodOPQRSTUV, vshift), _mm256_cmpgt_epi32(vremOPQRSTUV, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
@@ -356,8 +356,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -369,21 +369,21 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx512skx-mul32.c b/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx512skx-mul32.c
index 900c18b..9db9dbf 100644
--- a/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx512skx-mul32.c
+++ b/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-avx512skx-mul32.c
@@ -31,14 +31,14 @@
assert(output_width != 0);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 5, 1, 6, 2, 4, 0);
do {
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-neon-mul16.c b/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-neon-mul16.c
index 065b9fb..1e37af6 100644
--- a/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up32x9-minmax-gemmlowp-neon-mul16.c
@@ -29,12 +29,12 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up4x25-minmax-gemmlowp-scalar.c b/src/qs8-dwconv/gen/up4x25-minmax-gemmlowp-scalar.c
index b4232a6..850ca9b 100644
--- a/src/qs8-dwconv/gen/up4x25-minmax-gemmlowp-scalar.c
+++ b/src/qs8-dwconv/gen/up4x25-minmax-gemmlowp-scalar.c
@@ -28,14 +28,14 @@
assert(channels != 0);
assert(output_width != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vq31rounding = INT64_C(0x40000000);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up4x9-minmax-gemmlowp-scalar.c b/src/qs8-dwconv/gen/up4x9-minmax-gemmlowp-scalar.c
index 402518f..b6da6c7 100644
--- a/src/qs8-dwconv/gen/up4x9-minmax-gemmlowp-scalar.c
+++ b/src/qs8-dwconv/gen/up4x9-minmax-gemmlowp-scalar.c
@@ -28,14 +28,14 @@
assert(channels != 0);
assert(output_width != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vq31rounding = INT64_C(0x40000000);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx2-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx2-mul32.c
new file mode 100644
index 0000000..a8af432
--- /dev/null
+++ b/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx2-mul32.c
@@ -0,0 +1,501 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul32.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up8x25__avx2_mul32(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 8; c -= 8) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ i0 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+ i1 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ i2 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+ i3 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ i4 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+ i5 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ i6 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+ i7 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ i8 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ const __m256i vi9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i9));
+ const __m256i vk9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t))));
+ i9 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi9x01234567, vk9x01234567));
+
+ const __m256i vi10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i10));
+ const __m256i vk10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+ i10 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi10x01234567, vk10x01234567));
+
+ const __m256i vi11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i11));
+ const __m256i vk11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t))));
+ i11 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi11x01234567, vk11x01234567));
+
+ const __m256i vi12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i12));
+ const __m256i vk12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+ i12 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi12x01234567, vk12x01234567));
+
+ const __m256i vi13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i13));
+ const __m256i vk13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t))));
+ i13 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi13x01234567, vk13x01234567));
+
+ const __m256i vi14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i14));
+ const __m256i vk14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+ i14 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi14x01234567, vk14x01234567));
+
+ const __m256i vi15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i15));
+ const __m256i vk15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t))));
+ i15 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi15x01234567, vk15x01234567));
+
+ const __m256i vi16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i16));
+ const __m256i vk16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+ i16 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi16x01234567, vk16x01234567));
+
+ const __m256i vi17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i17));
+ const __m256i vk17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t))));
+ i17 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi17x01234567, vk17x01234567));
+
+ const __m256i vi18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i18));
+ const __m256i vk18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+ i18 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi18x01234567, vk18x01234567));
+
+ const __m256i vi19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i19));
+ const __m256i vk19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t))));
+ i19 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi19x01234567, vk19x01234567));
+
+ const __m256i vi20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i20));
+ const __m256i vk20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+ i20 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi20x01234567, vk20x01234567));
+
+ const __m256i vi21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i21));
+ const __m256i vk21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t))));
+ i21 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi21x01234567, vk21x01234567));
+
+ const __m256i vi22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i22));
+ const __m256i vk22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+ i22 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi22x01234567, vk22x01234567));
+
+ const __m256i vi23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i23));
+ const __m256i vk23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t))));
+ i23 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi23x01234567, vk23x01234567));
+
+ const __m256i vi24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i24));
+ const __m256i vk24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+ i24 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
+
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+
+ _mm_storel_epi64((__m128i*) output, vout0123456701234567);
+ output += 8;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ const __m256i vi9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i9));
+ const __m256i vk9x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi9x01234567, vk9x01234567));
+
+ const __m256i vi10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i10));
+ const __m256i vk10x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi10x01234567, vk10x01234567));
+
+ const __m256i vi11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i11));
+ const __m256i vk11x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi11x01234567, vk11x01234567));
+
+ const __m256i vi12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i12));
+ const __m256i vk12x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi12x01234567, vk12x01234567));
+
+ const __m256i vi13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i13));
+ const __m256i vk13x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi13x01234567, vk13x01234567));
+
+ const __m256i vi14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i14));
+ const __m256i vk14x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi14x01234567, vk14x01234567));
+
+ const __m256i vi15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i15));
+ const __m256i vk15x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi15x01234567, vk15x01234567));
+
+ const __m256i vi16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i16));
+ const __m256i vk16x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi16x01234567, vk16x01234567));
+
+ const __m256i vi17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i17));
+ const __m256i vk17x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi17x01234567, vk17x01234567));
+
+ const __m256i vi18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i18));
+ const __m256i vk18x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi18x01234567, vk18x01234567));
+
+ const __m256i vi19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i19));
+ const __m256i vk19x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi19x01234567, vk19x01234567));
+
+ const __m256i vi20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i20));
+ const __m256i vk20x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi20x01234567, vk20x01234567));
+
+ const __m256i vi21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i21));
+ const __m256i vk21x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi21x01234567, vk21x01234567));
+
+ const __m256i vi22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i22));
+ const __m256i vk22x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi22x01234567, vk22x01234567));
+
+ const __m256i vi23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i23));
+ const __m256i vk23x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi23x01234567, vk23x01234567));
+
+ const __m256i vi24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i24));
+ const __m256i vk24x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
+
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
+ vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456701234567, 0);
+ vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
+ output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul16.c
index 570bc8f..185b124 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul16.c
@@ -491,8 +491,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -509,29 +509,29 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
@@ -844,8 +844,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp24x01234567lo, vp24x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -864,27 +864,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul32.c
index 5f0c9bd..d9aa725 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul32.c
@@ -392,8 +392,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -410,24 +410,24 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
@@ -594,8 +594,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -607,21 +607,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx2-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx2-mul32.c
index 52c1b39..7734857 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx2-mul32.c
@@ -315,8 +315,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -328,22 +328,22 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
@@ -481,8 +481,8 @@
vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -494,21 +494,21 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-neon-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-neon-mul16.c
index 7c4e207..8a613da 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-neon-mul16.c
@@ -29,12 +29,12 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse2-mul16.c
index ab59747..de6b71d 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse2-mul16.c
@@ -491,8 +491,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -530,26 +530,26 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
@@ -865,8 +865,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp24x01234567lo, vp24x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -905,24 +905,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul16.c
index 13c53f9..d06566d 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul16.c
@@ -491,8 +491,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -509,29 +509,29 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
@@ -844,8 +844,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp24x01234567lo, vp24x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -864,27 +864,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul32.c
index 8b636e3..fba3b5e 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul32.c
@@ -392,8 +392,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -410,24 +410,24 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
@@ -594,8 +594,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -607,21 +607,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-ssse3-mul16.c
index 8e54b28..9d0875d 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-ssse3-mul16.c
@@ -491,8 +491,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -530,26 +530,26 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
@@ -865,8 +865,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp24x01234567lo, vp24x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -905,24 +905,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-wasmsimd-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-wasmsimd-mul16.c
index 0c39bd3..1e757fa 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-wasmsimd-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-wasmsimd-mul16.c
@@ -399,8 +399,8 @@
const v128_t vacc45 = wasm_v32x4_shuffle(vacc4567, vsign4567, 0, 4, 1, 5);
const v128_t vacc67 = wasm_v32x4_shuffle(vacc4567, vsign4567, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -409,20 +409,20 @@
const v128_t vq31prod0123 = wasm_v32x4_shuffle(vprod01, vprod23, 1, 3, 5, 7);
const v128_t vq31prod4567 = wasm_v32x4_shuffle(vprod45, vprod67, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
*((double*) output) = wasm_f64x2_extract_lane(vout0123456701234567, 0);
@@ -643,8 +643,8 @@
const v128_t vacc45 = wasm_v32x4_shuffle(vacc4567, vsign4567, 0, 4, 1, 5);
const v128_t vacc67 = wasm_v32x4_shuffle(vacc4567, vsign4567, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -653,20 +653,20 @@
const v128_t vq31prod0123 = wasm_v32x4_shuffle(vprod01, vprod23, 1, 3, 5, 7);
const v128_t vq31prod4567 = wasm_v32x4_shuffle(vprod45, vprod67, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
if (c & 4) {
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-xop-mul32.c
index 777be2d..a3d67a8 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-xop-mul32.c
@@ -397,8 +397,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -415,24 +415,24 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
@@ -599,8 +599,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -612,21 +612,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx2-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx2-mul32.c
new file mode 100644
index 0000000..fb18b6c
--- /dev/null
+++ b/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx2-mul32.c
@@ -0,0 +1,245 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-avx2-mul32.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_fp32_ukernel_up8x9__avx2_mul32(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 8; c -= 8) {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+ i0 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+ i1 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+ i2 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+ i3 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+ i4 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+ i5 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+ i6 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+ i7 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+ i8 += 8;
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
+
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+
+ _mm_storel_epi64((__m128i*) output, vout0123456701234567);
+ output += 8;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
+
+
+ const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
+ const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
+
+ const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
+ const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
+
+ const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
+ const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
+
+ const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
+ const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
+
+ const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
+ const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
+
+ const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
+ const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
+
+ const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
+ const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
+
+ const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
+ const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
+
+ const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
+ const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t))));
+
+ vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
+
+
+ __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
+ vscaled01234567 = _mm256_mul_ps(vscaled01234567, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_avx2.output_zero_point);
+ __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
+
+ __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ if (c & 4) {
+ *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
+ vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
+ output += 4;
+ }
+ if (c & 2) {
+ *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout0123456701234567, 0);
+ vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
+ output += 2;
+ }
+ if (c & 1) {
+ *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
+ output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul16.c
index 1f981e8..5ab356a 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul16.c
@@ -203,8 +203,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -221,29 +221,29 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
@@ -364,8 +364,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp8x01234567lo, vp8x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -384,27 +384,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul32.c
index ae1c6ca..d1362f2 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul32.c
@@ -168,8 +168,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -186,24 +186,24 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
@@ -274,8 +274,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -287,21 +287,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx2-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx2-mul32.c
index 9993035..5e039fe 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx2-mul32.c
@@ -139,8 +139,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,22 +152,22 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
@@ -225,8 +225,8 @@
vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -238,21 +238,21 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.output_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-neon-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-neon-mul16.c
index 9ee7b59..dfd5e06 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-neon-mul16.c
@@ -29,12 +29,12 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
const int8_t* i0 = input[0];
assert(i0 != NULL);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse2-mul16.c
index 862adf9..0f87944 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse2-mul16.c
@@ -203,8 +203,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -242,26 +242,26 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
@@ -385,8 +385,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp8x01234567lo, vp8x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -425,24 +425,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul16.c
index d844f6f..f6e8b72 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul16.c
@@ -203,8 +203,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -221,29 +221,29 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
@@ -364,8 +364,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp8x01234567lo, vp8x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -384,27 +384,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
- vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul32.c
index 70e4c87..ff41ad6 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul32.c
@@ -168,8 +168,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -186,24 +186,24 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
@@ -274,8 +274,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -287,21 +287,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-ssse3-mul16.c
index 1a7df5c..717701a 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-ssse3-mul16.c
@@ -203,8 +203,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -242,26 +242,26 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
@@ -385,8 +385,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp8x01234567lo, vp8x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0123);
const __m128i vnmask4567 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc4567);
@@ -425,24 +425,24 @@
const __m128i vq31prod0123 = _mm_shuffle_epi32(vq31prod0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod4567 = _mm_shuffle_epi32(vq31prod4657, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-wasmsimd-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-wasmsimd-mul16.c
index cef3330..de37634 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-wasmsimd-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-wasmsimd-mul16.c
@@ -175,8 +175,8 @@
const v128_t vacc45 = wasm_v32x4_shuffle(vacc4567, vsign4567, 0, 4, 1, 5);
const v128_t vacc67 = wasm_v32x4_shuffle(vacc4567, vsign4567, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -185,20 +185,20 @@
const v128_t vq31prod0123 = wasm_v32x4_shuffle(vprod01, vprod23, 1, 3, 5, 7);
const v128_t vq31prod4567 = wasm_v32x4_shuffle(vprod45, vprod67, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
*((double*) output) = wasm_f64x2_extract_lane(vout0123456701234567, 0);
@@ -291,8 +291,8 @@
const v128_t vacc45 = wasm_v32x4_shuffle(vacc4567, vsign4567, 0, 4, 1, 5);
const v128_t vacc67 = wasm_v32x4_shuffle(vacc4567, vsign4567, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod01 = wasm_i64x2_add(wasm_i64x2_mul(vacc01, vmultiplier), vrounding);
const v128_t vprod23 = wasm_i64x2_add(wasm_i64x2_mul(vacc23, vmultiplier), vrounding);
const v128_t vprod45 = wasm_i64x2_add(wasm_i64x2_mul(vacc45, vmultiplier), vrounding);
@@ -301,20 +301,20 @@
const v128_t vq31prod0123 = wasm_v32x4_shuffle(vprod01, vprod23, 1, 3, 5, 7);
const v128_t vq31prod4567 = wasm_v32x4_shuffle(vprod45, vprod67, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0123, vremainder_mask), wasm_i32x4_shr(vq31prod0123, 31));
const v128_t vrem4567 = wasm_i32x4_add(wasm_v128_and(vq31prod4567, vremainder_mask), wasm_i32x4_shr(vq31prod4567, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0123, vshift), wasm_i32x4_gt(vrem0123, vthreshold));
vacc4567 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod4567, vshift), wasm_i32x4_gt(vrem4567, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout01234567 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout0123456701234567 = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout01234567, vout01234567), voutput_min), voutput_max);
if (c & 4) {
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-xop-mul32.c
index 943bea2..c1bede9 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-xop-mul32.c
@@ -173,8 +173,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -191,24 +191,24 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
@@ -279,8 +279,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -292,21 +292,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
vout0123 = _mm_packs_epi16(vout0123, vout0123);
- vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/unipass-avx2-mul16.c.in b/src/qs8-dwconv/unipass-avx2-mul16.c.in
index a6b5be5..e01b67e 100644
--- a/src/qs8-dwconv/unipass-avx2-mul16.c.in
+++ b/src/qs8-dwconv/unipass-avx2-mul16.c.in
@@ -4,6 +4,7 @@
// LICENSE file in the root directory of this source tree.
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert CHANNEL_TILE % 16 == 0
$assert CHANNEL_TILE >= 16
$assert KERNEL_TILE >= 2
@@ -14,7 +15,7 @@
#include <xnnpack/dwconv.h>
-void xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__avx2_mul16(
+void xnn_qs8_dwconv_minmax_${REQUANTIZATION.lower()}_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__avx2_mul16(
size_t channels,
size_t output_width,
const int8_t** input,
@@ -62,47 +63,57 @@
vacc${ABC[C+8:C+16]} = _mm256_add_epi32(vacc${ABC[C+8:C+16]}, _mm256_cvtepi16_epi32(vprod${K}x${ABC[C+8:C+16]}));
w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${KERNEL_TILE * CHANNEL_TILE} * sizeof(int8_t));
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vacc${ABC[C+1:C+8:2]} = _mm256_shuffle_epi32(vacc${ABC[C:C+8]}, _MM_SHUFFLE(3, 3, 1, 1));
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vacc${ABC[C+1:C+8:2]} = _mm256_shuffle_epi32(vacc${ABC[C:C+8]}, _MM_SHUFFLE(3, 3, 1, 1));
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vprod${ABC[C:C+8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[C:C+8]}, vmultiplier), vrounding);
+ const __m256i vprod${ABC[C+1:C+8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[C+1:C+8:2]}, vmultiplier), vrounding);
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vprod${ABC[C:C+8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[C:C+8]}, vmultiplier), vrounding);
- const __m256i vprod${ABC[C+1:C+8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[C+1:C+8:2]}, vmultiplier), vrounding);
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vq31prod${ABC[C:C+8:2]} = _mm256_srli_epi64(vprod${ABC[C:C+8:2]}, 31);
+ const __m256i vq31prod${ABC[C+1:C+8:2]} = _mm256_add_epi64(vprod${ABC[C+1:C+8:2]}, vprod${ABC[C+1:C+8:2]});
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vq31prod${ABC[C:C+8:2]} = _mm256_srli_epi64(vprod${ABC[C:C+8:2]}, 31);
- const __m256i vq31prod${ABC[C+1:C+8:2]} = _mm256_add_epi64(vprod${ABC[C+1:C+8:2]}, vprod${ABC[C+1:C+8:2]});
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vq31prod${ABC[C:C+8]} = _mm256_blend_epi16(vq31prod${ABC[C:C+8:2]}, vq31prod${ABC[C+1:C+8:2]}, 0xCC);
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vq31prod${ABC[C:C+8]} = _mm256_blend_epi16(vq31prod${ABC[C:C+8:2]}, vq31prod${ABC[C+1:C+8:2]}, 0xCC);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vrem${ABC[C:C+8]} =
+ _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[C:C+8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[C:C+8]}));
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vrem${ABC[C:C+8]} =
- _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[C:C+8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[C:C+8]}));
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ $if CHANNEL_TILE > 8:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
+ $for C in range(0, CHANNEL_TILE, 8):
+ vacc${ABC[C:C+8]} =
+ _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[C:C+8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[C:C+8]}, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for C in range(0, CHANNEL_TILE, 8):
+ __m256 vscaled${ABC[C:C+8]} = _mm256_cvtepi32_ps(vacc${ABC[C:C+8]});
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- $if CHANNEL_TILE > 8:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
- $for C in range(0, CHANNEL_TILE, 8):
- vacc${ABC[C:C+8]} =
- _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[C:C+8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[C:C+8]}, vremainder_threshold));
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ $for C in range(0, CHANNEL_TILE, 8):
+ vscaled${ABC[C:C+8]} = _mm256_mul_ps(vscaled${ABC[C:C+8]}, vscale);
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ $for C in range(0, CHANNEL_TILE, 8):
+ vacc${ABC[C:C+8]} = _mm256_cvtps_epi32(vscaled${ABC[C:C+8]});
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_zero_point);
$for C in range(0, CHANNEL_TILE, 16):
const __m256i vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]} = _mm256_adds_epi16(_mm256_packs_epi32(vacc${ABC[C:C+8]}, vacc${ABC[C+8:C+16]}), voutput_zero_point);
$for C in range(0, CHANNEL_TILE, 16):
__m128i vout${ABC[C:C+16]} = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]}), _mm256_extracti128_si256(vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]}, 1)), _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_max);
$for C in range(0, CHANNEL_TILE, 16):
vout${ABC[C:C+16]} = _mm_max_epi8(vout${ABC[C:C+16]}, voutput_min);
vout${ABC[C:C+16]} = _mm_min_epi8(vout${ABC[C:C+16]}, voutput_max);
@@ -141,44 +152,55 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
k += 16;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
- const __m256i vacc${ABC[1:8:2]} = _mm256_shuffle_epi32(vacc${ABC[0:8]}, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vacc${ABC[9:16:2]} = _mm256_shuffle_epi32(vacc${ABC[8:16]}, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc${ABC[1:8:2]} = _mm256_shuffle_epi32(vacc${ABC[0:8]}, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc${ABC[9:16:2]} = _mm256_shuffle_epi32(vacc${ABC[8:16]}, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vprod${ABC[0:8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[0:8]}, vmultiplier), vrounding);
- const __m256i vprod${ABC[1:8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[1:8:2]}, vmultiplier), vrounding);
- const __m256i vprod${ABC[8:16:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[8:16]}, vmultiplier), vrounding);
- const __m256i vprod${ABC[9:16:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[9:16:2]}, vmultiplier), vrounding);
+ const __m256i vprod${ABC[0:8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[0:8]}, vmultiplier), vrounding);
+ const __m256i vprod${ABC[1:8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[1:8:2]}, vmultiplier), vrounding);
+ const __m256i vprod${ABC[8:16:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[8:16]}, vmultiplier), vrounding);
+ const __m256i vprod${ABC[9:16:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[9:16:2]}, vmultiplier), vrounding);
- const __m256i vq31prod${ABC[0:8:2]} = _mm256_srli_epi64(vprod${ABC[0:8:2]}, 31);
- const __m256i vq31prod${ABC[1:8:2]} = _mm256_add_epi64(vprod${ABC[1:8:2]}, vprod${ABC[1:8:2]});
- const __m256i vq31prod${ABC[8:16:2]} = _mm256_srli_epi64(vprod${ABC[8:16:2]}, 31);
- const __m256i vq31prod${ABC[9:16:2]} = _mm256_add_epi64(vprod${ABC[9:16:2]}, vprod${ABC[9:16:2]});
+ const __m256i vq31prod${ABC[0:8:2]} = _mm256_srli_epi64(vprod${ABC[0:8:2]}, 31);
+ const __m256i vq31prod${ABC[1:8:2]} = _mm256_add_epi64(vprod${ABC[1:8:2]}, vprod${ABC[1:8:2]});
+ const __m256i vq31prod${ABC[8:16:2]} = _mm256_srli_epi64(vprod${ABC[8:16:2]}, 31);
+ const __m256i vq31prod${ABC[9:16:2]} = _mm256_add_epi64(vprod${ABC[9:16:2]}, vprod${ABC[9:16:2]});
- const __m256i vq31prod${ABC[0:8]} = _mm256_blend_epi16(vq31prod${ABC[0:8:2]}, vq31prod${ABC[1:8:2]}, 0xCC);
- const __m256i vq31prod${ABC[8:16]} = _mm256_blend_epi16(vq31prod${ABC[8:16:2]}, vq31prod${ABC[9:16:2]}, 0xCC);
+ const __m256i vq31prod${ABC[0:8]} = _mm256_blend_epi16(vq31prod${ABC[0:8:2]}, vq31prod${ABC[1:8:2]}, 0xCC);
+ const __m256i vq31prod${ABC[8:16]} = _mm256_blend_epi16(vq31prod${ABC[8:16:2]}, vq31prod${ABC[9:16:2]}, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
- const __m256i vrem${ABC[0:8]} =
- _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[0:8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[0:8]}));
- const __m256i vrem${ABC[8:16]} =
- _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[8:16]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[8:16]}));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
+ const __m256i vrem${ABC[0:8]} =
+ _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[0:8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[0:8]}));
+ const __m256i vrem${ABC[8:16]} =
+ _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[8:16]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[8:16]}));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
- vacc${ABC[0:8]} =
- _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[0:8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[0:8]}, vremainder_threshold));
- vacc${ABC[8:16]} =
- _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[8:16]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[8:16]}, vremainder_threshold));
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
+ vacc${ABC[0:8]} =
+ _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[0:8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[0:8]}, vremainder_threshold));
+ vacc${ABC[8:16]} =
+ _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[8:16]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[8:16]}, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ __m256 vscaled${ABC[0:8]} = _mm256_cvtepi32_ps(vacc${ABC[0:8]});
+ __m256 vscaled${ABC[8:16]} = _mm256_cvtepi32_ps(vacc${ABC[8:16]});
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled${ABC[0:8]} = _mm256_mul_ps(vscaled${ABC[0:8]}, vscale);
+ vscaled${ABC[8:16]} = _mm256_mul_ps(vscaled${ABC[8:16]}, vscale);
+
+ vacc${ABC[0:8]} = _mm256_cvtps_epi32(vscaled${ABC[0:8]});
+ vacc${ABC[8:16]} = _mm256_cvtps_epi32(vscaled${ABC[8:16]});
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_zero_point);
__m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc${ABC[0:8]}), _mm256_extracti128_si256(vacc${ABC[0:8]}, 1)), voutput_zero_point);
__m128i vout${ABC[8:16]} = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc${ABC[8:16]}), _mm256_extracti128_si256(vacc${ABC[8:16]}, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_max);
__m128i vout${ABC[0:16]} = _mm_packs_epi16(vout${ABC[0:8]}, vout${ABC[8:16]});
vout${ABC[0:16]} = _mm_min_epi8(_mm_max_epi8(vout${ABC[0:16]}, voutput_min), voutput_max);
diff --git a/src/qs8-dwconv/unipass-avx2-mul32.c.in b/src/qs8-dwconv/unipass-avx2-mul32.c.in
index 8a031be..8f6d839 100644
--- a/src/qs8-dwconv/unipass-avx2-mul32.c.in
+++ b/src/qs8-dwconv/unipass-avx2-mul32.c.in
@@ -4,6 +4,7 @@
// LICENSE file in the root directory of this source tree.
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert CHANNEL_TILE % 8 == 0
$assert CHANNEL_TILE >= 8
$assert KERNEL_TILE >= 2
@@ -14,7 +15,7 @@
#include <xnnpack/dwconv.h>
-void xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__avx2_mul32(
+void xnn_qs8_dwconv_minmax_${REQUANTIZATION.lower()}_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__avx2_mul32(
size_t channels,
size_t output_width,
const int8_t** input,
@@ -60,41 +61,52 @@
w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${KERNEL_TILE * CHANNEL_TILE} * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vacc${ABC[C+1:C+8:2]} = _mm256_shuffle_epi32(vacc${ABC[C:C+8]}, _MM_SHUFFLE(3, 3, 1, 1));
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vacc${ABC[C+1:C+8:2]} = _mm256_shuffle_epi32(vacc${ABC[C:C+8]}, _MM_SHUFFLE(3, 3, 1, 1));
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vprod${ABC[C:C+8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[C:C+8]}, vmultiplier), vrounding);
- const __m256i vprod${ABC[C+1:C+8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[C+1:C+8:2]}, vmultiplier), vrounding);
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vprod${ABC[C:C+8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[C:C+8]}, vmultiplier), vrounding);
+ const __m256i vprod${ABC[C+1:C+8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[C+1:C+8:2]}, vmultiplier), vrounding);
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vq31prod${ABC[C:C+8:2]} = _mm256_srli_epi64(vprod${ABC[C:C+8:2]}, 31);
- const __m256i vq31prod${ABC[C+1:C+8:2]} = _mm256_add_epi64(vprod${ABC[C+1:C+8:2]}, vprod${ABC[C+1:C+8:2]});
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vq31prod${ABC[C:C+8:2]} = _mm256_srli_epi64(vprod${ABC[C:C+8:2]}, 31);
+ const __m256i vq31prod${ABC[C+1:C+8:2]} = _mm256_add_epi64(vprod${ABC[C+1:C+8:2]}, vprod${ABC[C+1:C+8:2]});
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vq31prod${ABC[C:C+8]} = _mm256_blend_epi16(vq31prod${ABC[C:C+8:2]}, vq31prod${ABC[C+1:C+8:2]}, 0xCC);
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vq31prod${ABC[C:C+8]} = _mm256_blend_epi16(vq31prod${ABC[C:C+8:2]}, vq31prod${ABC[C+1:C+8:2]}, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
- $for C in range(0, CHANNEL_TILE, 8):
- const __m256i vrem${ABC[C:C+8]} =
- _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[C:C+8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[C:C+8]}));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
+ $for C in range(0, CHANNEL_TILE, 8):
+ const __m256i vrem${ABC[C:C+8]} =
+ _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[C:C+8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[C:C+8]}));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- $if CHANNEL_TILE > 8:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
- $for C in range(0, CHANNEL_TILE, 8):
- vacc${ABC[C:C+8]} =
- _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[C:C+8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[C:C+8]}, vremainder_threshold));
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ $if CHANNEL_TILE > 8:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
+ $for C in range(0, CHANNEL_TILE, 8):
+ vacc${ABC[C:C+8]} =
+ _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[C:C+8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[C:C+8]}, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for C in range(0, CHANNEL_TILE, 8):
+ __m256 vscaled${ABC[C:C+8]} = _mm256_cvtepi32_ps(vacc${ABC[C:C+8]});
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ $for C in range(0, CHANNEL_TILE, 8):
+ vscaled${ABC[C:C+8]} = _mm256_mul_ps(vscaled${ABC[C:C+8]}, vscale);
+
+ $for C in range(0, CHANNEL_TILE, 8):
+ vacc${ABC[C:C+8]} = _mm256_cvtps_epi32(vscaled${ABC[C:C+8]});
$if CHANNEL_TILE > 8:
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_zero_point);
$else:
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_zero_point);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
__m256i vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]} = _mm256_adds_epi16(_mm256_packs_epi32(vacc${ABC[C:C+8]}, vacc${ABC[C+8:C+16]}), voutput_zero_point);
@@ -109,8 +121,8 @@
$else:
__m128i vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C:C+8]});
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_max);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
vout${ABC[C:C+16]} = _mm_max_epi8(vout${ABC[C:C+16]}, voutput_min);
@@ -155,34 +167,39 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
- const __m256i vacc${ABC[1:8:2]} = _mm256_shuffle_epi32(vacc${ABC[0:8]}, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc${ABC[1:8:2]} = _mm256_shuffle_epi32(vacc${ABC[0:8]}, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vprod${ABC[0:8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[0:8]}, vmultiplier), vrounding);
- const __m256i vprod${ABC[1:8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[1:8:2]}, vmultiplier), vrounding);
+ const __m256i vprod${ABC[0:8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[0:8]}, vmultiplier), vrounding);
+ const __m256i vprod${ABC[1:8:2]} = _mm256_add_epi64(_mm256_mul_epi32(vacc${ABC[1:8:2]}, vmultiplier), vrounding);
- const __m256i vq31prod${ABC[0:8:2]} = _mm256_srli_epi64(vprod${ABC[0:8:2]}, 31);
- const __m256i vq31prod${ABC[1:8:2]} = _mm256_add_epi64(vprod${ABC[1:8:2]}, vprod${ABC[1:8:2]});
+ const __m256i vq31prod${ABC[0:8:2]} = _mm256_srli_epi64(vprod${ABC[0:8:2]}, 31);
+ const __m256i vq31prod${ABC[1:8:2]} = _mm256_add_epi64(vprod${ABC[1:8:2]}, vprod${ABC[1:8:2]});
- const __m256i vq31prod${ABC[0:8]} = _mm256_blend_epi16(vq31prod${ABC[0:8:2]}, vq31prod${ABC[1:8:2]}, 0xCC);
+ const __m256i vq31prod${ABC[0:8]} = _mm256_blend_epi16(vq31prod${ABC[0:8:2]}, vq31prod${ABC[1:8:2]}, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
- const __m256i vrem${ABC[0:8]} =
- _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[0:8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[0:8]}));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
+ const __m256i vrem${ABC[0:8]} =
+ _mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[0:8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[0:8]}));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
- vacc${ABC[0:8]} =
- _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[0:8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[0:8]}, vremainder_threshold));
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
+ vacc${ABC[0:8]} =
+ _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[0:8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[0:8]}, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ __m256 vscaled${ABC[0:8]} = _mm256_cvtepi32_ps(vacc${ABC[0:8]});
+ vscaled${ABC[0:8]} = _mm256_mul_ps(vscaled${ABC[0:8]}, _mm256_load_ps(params->fp32_avx2.scale));
+ vacc${ABC[0:8]} = _mm256_cvtps_epi32(vscaled${ABC[0:8]});
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_zero_point);
__m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc${ABC[0:8]}), _mm256_extracti128_si256(vacc${ABC[0:8]}, 1)), voutput_zero_point);
__m128i vout${ABC[0:8]}${ABC[0:8]} = _mm_packs_epi16(vout${ABC[0:8]}, vout${ABC[0:8]});
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${REQUANTIZATION.lower()}_avx2.output_min);
vout${ABC[0:8]}${ABC[0:8]} = _mm_min_epi8(vout${ABC[0:8]}${ABC[0:8]}, voutput_max);
vout${ABC[0:8]}${ABC[0:8]} = _mm_max_epi8(vout${ABC[0:8]}${ABC[0:8]}, voutput_min);
diff --git a/src/qs8-dwconv/unipass-avx512skx-mul32.c.in b/src/qs8-dwconv/unipass-avx512skx-mul32.c.in
index 1a8cf8f..71e6ddb 100644
--- a/src/qs8-dwconv/unipass-avx512skx-mul32.c.in
+++ b/src/qs8-dwconv/unipass-avx512skx-mul32.c.in
@@ -31,20 +31,20 @@
assert(output_width != 0);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
$if CHANNEL_TILE > 16:
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 5, 1, 6, 2, 4, 0);
$else:
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
do {
$for K in range(KERNEL_TILE):
diff --git a/src/qs8-dwconv/unipass-neon-mul16.c.in b/src/qs8-dwconv/unipass-neon-mul16.c.in
index ef0a0dc..47a7832 100644
--- a/src/qs8-dwconv/unipass-neon-mul16.c.in
+++ b/src/qs8-dwconv/unipass-neon-mul16.c.in
@@ -29,16 +29,16 @@
assert(channels != 0);
assert(output_width != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
$if CHANNEL_TILE == 8:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
do {
$for K in range(KERNEL_TILE):
const int8_t* i${K} = input[${K}];
diff --git a/src/qs8-dwconv/unipass-scalar.c.in b/src/qs8-dwconv/unipass-scalar.c.in
index 8881d15..268d5de 100644
--- a/src/qs8-dwconv/unipass-scalar.c.in
+++ b/src/qs8-dwconv/unipass-scalar.c.in
@@ -25,14 +25,14 @@
assert(channels != 0);
assert(output_width != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vq31rounding = INT64_C(0x40000000);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
do {
$for K in range(KERNEL_TILE):
const int8_t* i${K} = input[${K}];
diff --git a/src/qs8-dwconv/unipass-sse-mul16.c.in b/src/qs8-dwconv/unipass-sse-mul16.c.in
index 011c123..a367104 100644
--- a/src/qs8-dwconv/unipass-sse-mul16.c.in
+++ b/src/qs8-dwconv/unipass-sse-mul16.c.in
@@ -15,7 +15,7 @@
#include <xnnpack/dwconv.h>
-$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
+$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
$ISA = "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__${ISA}_mul16(
size_t channels,
@@ -142,11 +142,11 @@
__m128i vout${ABC[C:C+8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
$for C in range(0, CHANNEL_TILE, 8):
vout${ABC[C:C+8]} = _mm_max_epi16(vout${ABC[C:C+8]}, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
$for C in range(0, CHANNEL_TILE, 8):
vout${ABC[C:C+8]} = _mm_min_epi16(vout${ABC[C:C+8]}, voutput_max);
@@ -157,14 +157,14 @@
__m128i vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C:C+8]});
$if SSE == 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
vout${ABC[C:C+16]} = _mm_max_epi8(vout${ABC[C:C+16]}, voutput_min);
$else:
vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_max_epi8(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
vout${ABC[C:C+16]} = _mm_min_epi8(vout${ABC[C:C+16]}, voutput_max);
@@ -299,14 +299,14 @@
__m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[4:8]}), voutput_zero_point);
$if SSE < 4:
- vout${ABC[0:8]} = _mm_max_epi16(vout${ABC[0:8]}, _mm_load_si128((const __m128i*) params->sse2.output_min));
- vout${ABC[0:8]} = _mm_min_epi16(vout${ABC[0:8]}, _mm_load_si128((const __m128i*) params->sse2.output_max));
+ vout${ABC[0:8]} = _mm_max_epi16(vout${ABC[0:8]}, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min));
+ vout${ABC[0:8]} = _mm_min_epi16(vout${ABC[0:8]}, _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max));
__m128i vout${ABC[0:8]}${ABC[0:8]} = _mm_packs_epi16(vout${ABC[0:8]}, vout${ABC[0:8]});
$if SSE == 4:
- vout${ABC[0:8]}${ABC[0:8]} = _mm_max_epi8(vout${ABC[0:8]}${ABC[0:8]}, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout${ABC[0:8]}${ABC[0:8]} = _mm_min_epi8(vout${ABC[0:8]}${ABC[0:8]}, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout${ABC[0:8]}${ABC[0:8]} = _mm_max_epi8(vout${ABC[0:8]}${ABC[0:8]}, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout${ABC[0:8]}${ABC[0:8]} = _mm_min_epi8(vout${ABC[0:8]}${ABC[0:8]}, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
$if CHANNEL_TILE > 8:
if XNN_LIKELY(c >= 8) {
diff --git a/src/qs8-dwconv/unipass-sse-mul32.c.in b/src/qs8-dwconv/unipass-sse-mul32.c.in
index df8f462..82bd994 100644
--- a/src/qs8-dwconv/unipass-sse-mul32.c.in
+++ b/src/qs8-dwconv/unipass-sse-mul32.c.in
@@ -75,8 +75,8 @@
w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${KERNEL_TILE * CHANNEL_TILE} * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
$for C in range(0, CHANNEL_TILE, 4):
const __m128i vacc${ABC[C+1:C+4:2]} = _mm_shuffle_epi32(vacc${ABC[C:C+4]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -90,23 +90,23 @@
$for C in range(0, CHANNEL_TILE, 4):
const __m128i vq31prod${ABC[C:C+4]} = _mm_blend_epi16(vq31prod${ABC[C:C+4:2]}, vq31prod${ABC[C+1:C+4:2]}, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
$for C in range(0, CHANNEL_TILE, 4):
const __m128i vrem${ABC[C:C+4]} =
_mm_add_epi32(_mm_and_si128(vq31prod${ABC[C:C+4]}, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${ABC[C:C+4]}));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
$for C in range(0, CHANNEL_TILE, 4):
vacc${ABC[C:C+4]} =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${ABC[C:C+4]}, vshift), _mm_cmpgt_epi32(vrem${ABC[C:C+4]}, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
$for C in range(0, CHANNEL_TILE, 8):
__m128i vout${ABC[C:C+8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
__m128i vout${ABC[C:C+16]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C+8:C+16]});
@@ -156,8 +156,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc${ABC[1:4:2]} = _mm_shuffle_epi32(vacc${ABC[0:4]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -169,21 +169,21 @@
const __m128i vq31prod${ABC[0:4]} = _mm_blend_epi16(vq31prod${ABC[0:4:2]}, vq31prod${ABC[1:4:2]}, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem${ABC[0:4]} =
_mm_add_epi32(_mm_and_si128(vq31prod${ABC[0:4]}, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${ABC[0:4]}));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc${ABC[0:4]} =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${ABC[0:4]}, vshift), _mm_cmpgt_epi32(vrem${ABC[0:4]}, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout${ABC[0:4]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[0:4]}), voutput_zero_point);
vout${ABC[0:4]} = _mm_packs_epi16(vout${ABC[0:4]}, vout${ABC[0:4]});
- vout${ABC[0:4]} = _mm_max_epi8(vout${ABC[0:4]}, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout${ABC[0:4]} = _mm_min_epi8(vout${ABC[0:4]}, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout${ABC[0:4]} = _mm_max_epi8(vout${ABC[0:4]}, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout${ABC[0:4]} = _mm_min_epi8(vout${ABC[0:4]}, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
$if CHANNEL_TILE > 4:
if XNN_LIKELY(c >= 4) {
diff --git a/src/qs8-dwconv/unipass-wasmsimd-mul16.c.in b/src/qs8-dwconv/unipass-wasmsimd-mul16.c.in
index d32f78b..db5295b 100644
--- a/src/qs8-dwconv/unipass-wasmsimd-mul16.c.in
+++ b/src/qs8-dwconv/unipass-wasmsimd-mul16.c.in
@@ -71,29 +71,29 @@
const v128_t vacc${ABC[C:C+2]} = wasm_v32x4_shuffle(vacc${ABC[C:C+4]}, vsign${ABC[C:C+4]}, 0, 4, 1, 5);
const v128_t vacc${ABC[C+2:C+4]} = wasm_v32x4_shuffle(vacc${ABC[C:C+4]}, vsign${ABC[C:C+4]}, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
$for C in range(0, CHANNEL_TILE, 2):
const v128_t vprod${ABC[C:C+2]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[C:C+2]}, vmultiplier), vrounding);
$for C in range(0, CHANNEL_TILE, 4):
const v128_t vq31prod${ABC[C:C+4]} = wasm_v32x4_shuffle(vprod${ABC[C:C+2]}, vprod${ABC[C+2:C+4]}, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
$for C in range(0, CHANNEL_TILE, 4):
const v128_t vrem${ABC[C:C+4]} = wasm_i32x4_add(wasm_v128_and(vq31prod${ABC[C:C+4]}, vremainder_mask), wasm_i32x4_shr(vq31prod${ABC[C:C+4]}, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
$for C in range(0, CHANNEL_TILE, 4):
vacc${ABC[C:C+4]} = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${ABC[C:C+4]}, vshift), wasm_i32x4_gt(vrem${ABC[C:C+4]}, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
$for C in range(0, CHANNEL_TILE, 8):
v128_t vout${ABC[C:C+8]} = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
v128_t vout${ABC[C:C+16]} = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout${ABC[C:C+8]}, vout${ABC[C+8:C+16]}), voutput_min), voutput_max);
@@ -148,8 +148,8 @@
const v128_t vacc${ABC[4:6]} = wasm_v32x4_shuffle(vacc${ABC[4:8]}, vsign${ABC[4:8]}, 0, 4, 1, 5);
const v128_t vacc${ABC[6:8]} = wasm_v32x4_shuffle(vacc${ABC[4:8]}, vsign${ABC[4:8]}, 2, 6, 3, 7);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod${ABC[0:2]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[0:2]}, vmultiplier), vrounding);
const v128_t vprod${ABC[2:4]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[2:4]}, vmultiplier), vrounding);
const v128_t vprod${ABC[4:6]} = wasm_i64x2_add(wasm_i64x2_mul(vacc${ABC[4:6]}, vmultiplier), vrounding);
@@ -158,20 +158,20 @@
const v128_t vq31prod${ABC[0:4]} = wasm_v32x4_shuffle(vprod${ABC[0:2]}, vprod${ABC[2:4]}, 1, 3, 5, 7);
const v128_t vq31prod${ABC[4:8]} = wasm_v32x4_shuffle(vprod${ABC[4:6]}, vprod${ABC[6:8]}, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem${ABC[0:4]} = wasm_i32x4_add(wasm_v128_and(vq31prod${ABC[0:4]}, vremainder_mask), wasm_i32x4_shr(vq31prod${ABC[0:4]}, 31));
const v128_t vrem${ABC[4:8]} = wasm_i32x4_add(wasm_v128_and(vq31prod${ABC[4:8]}, vremainder_mask), wasm_i32x4_shr(vq31prod${ABC[4:8]}, 31));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc${ABC[0:4]} = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${ABC[0:4]}, vshift), wasm_i32x4_gt(vrem${ABC[0:4]}, vthreshold));
vacc${ABC[4:8]} = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${ABC[4:8]}, vshift), wasm_i32x4_gt(vrem${ABC[4:8]}, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vout${ABC[0:8]} = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc${ABC[0:4]}, vacc${ABC[4:8]}), voutput_zero_point);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
v128_t vout${ABC[0:8]}${ABC[0:8]} = wasm_i8x16_min(wasm_i8x16_max(wasm_i8x16_narrow_i16x8(vout${ABC[0:8]}, vout${ABC[0:8]}), voutput_min), voutput_max);
$if CHANNEL_TILE > 8:
diff --git a/src/qs8-gemm/MRx16c8-avx512skx.c.in b/src/qs8-gemm/MRx16c8-avx512skx.c.in
index 9b842bc..a2d725d 100644
--- a/src/qs8-gemm/MRx16c8-avx512skx.c.in
+++ b/src/qs8-gemm/MRx16c8-avx512skx.c.in
@@ -61,24 +61,24 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
$if MR > 1:
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
$else:
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
$if MR > 2:
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
$elif MR == 2:
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
$else:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
$for N in range(4, 16, 4):
diff --git a/src/qs8-gemm/MRx4c2-sse.c.in b/src/qs8-gemm/MRx4c2-sse.c.in
index 77306ed..cdd7e71 100644
--- a/src/qs8-gemm/MRx4c2-sse.c.in
+++ b/src/qs8-gemm/MRx4c2-sse.c.in
@@ -27,7 +27,7 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
+$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x4c2__${ISA}${LOAD_SUFFIX}(
size_t mr,
@@ -297,8 +297,8 @@
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -308,8 +308,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/MRx4c8-sse.c.in b/src/qs8-gemm/MRx4c8-sse.c.in
index 656f725..6f26c05 100644
--- a/src/qs8-gemm/MRx4c8-sse.c.in
+++ b/src/qs8-gemm/MRx4c8-sse.c.in
@@ -27,7 +27,7 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
+$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x4c8__${ISA}${LOAD_SUFFIX}(
size_t mr,
@@ -234,8 +234,8 @@
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -245,8 +245,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/MRx4c8-wasmsimd.c.in b/src/qs8-gemm/MRx4c8-wasmsimd.c.in
index c91c1cc..14ec68d 100644
--- a/src/qs8-gemm/MRx4c8-wasmsimd.c.in
+++ b/src/qs8-gemm/MRx4c8-wasmsimd.c.in
@@ -131,8 +131,8 @@
$for M in range(MR):
const v128_t vacc${M}x01 = wasm_v32x4_shuffle(vacc${M}x0123, vsign${M}x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
$for M in range(MR):
const v128_t vprod${M}x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc${M}x01, vmultiplier), vrounding);
const v128_t vacc${M}x23 = wasm_v32x4_shuffle(vacc${M}x0123, vsign${M}x0123, 2, 6, 3, 7);
@@ -143,16 +143,16 @@
$for M in range(MR):
const v128_t vq31prod${M}x0123 = wasm_v32x4_shuffle(vprod${M}x01, vprod${M}x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
$for M in range(MR):
const v128_t vrem${M}x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod${M}x0123, vremainder_mask), wasm_i32x4_lt(vq31prod${M}x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
$for M in range(MR):
vacc${M}x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${M}x0123, vshift), wasm_i32x4_gt(vrem${M}x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
$for M in range(0, MR, 2):
v128_t vacc${M}${min(M+1, MR-1)}x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
@@ -161,10 +161,10 @@
$else:
v128_t vout = wasm_i8x16_narrow_i16x8(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/MRx8c8-avx2.c.in b/src/qs8-gemm/MRx8c8-avx2.c.in
index 761da0f..f92a9aa 100644
--- a/src/qs8-gemm/MRx8c8-avx2.c.in
+++ b/src/qs8-gemm/MRx8c8-avx2.c.in
@@ -3,6 +3,7 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD128", "EXTENDED"]
$assert MR <= 4
#include <assert.h>
@@ -15,7 +16,7 @@
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x8c8__avx2(
+void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x8c8__avx2(
size_t mr,
size_t nc,
size_t kc,
@@ -112,40 +113,51 @@
$for M in range(MR):
__m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
- $for M in range(MR):
- const __m256i vacc${M}x11335577 = _mm256_srli_epi64(vacc${M}x01234567, 32);
+ $for M in range(MR):
+ const __m256i vacc${M}x11335577 = _mm256_srli_epi64(vacc${M}x01234567, 32);
- $for M in range(MR):
- const __m256i vprod${M}x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x01234567, vmultiplier), vrounding);
+ $for M in range(MR):
+ const __m256i vprod${M}x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x01234567, vmultiplier), vrounding);
- $for M in range(MR):
- const __m256i vprod${M}x1357 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x11335577, vmultiplier), vrounding);
+ $for M in range(MR):
+ const __m256i vprod${M}x1357 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x11335577, vmultiplier), vrounding);
- $for M in range(MR):
- const __m256i vq31prod${M}x0246 = _mm256_srli_epi64(vprod${M}x0246, 31);
- const __m256i vq31prod${M}x1357 = _mm256_add_epi64(vprod${M}x1357, vprod${M}x1357);
+ $for M in range(MR):
+ const __m256i vq31prod${M}x0246 = _mm256_srli_epi64(vprod${M}x0246, 31);
+ const __m256i vq31prod${M}x1357 = _mm256_add_epi64(vprod${M}x1357, vprod${M}x1357);
- $for M in range(MR):
- const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
+ $for M in range(MR):
+ const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
- $for M in range(MR):
- const __m256i vrem${M}x01234567 =
- _mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
+ $for M in range(MR):
+ const __m256i vrem${M}x01234567 =
+ _mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
- $for M in range(MR):
- vacc${M}x01234567 =
- _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
+ $for M in range(MR):
+ vacc${M}x01234567 =
+ _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m256 vscaled${M}x01234567 = _mm256_cvtepi32_ps(vacc${M}x01234567);
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ $for M in range(MR):
+ vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale);
+
+ $for M in range(MR):
+ vacc${M}x01234567 = _mm256_cvtps_epi32(vscaled${M}x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_zero_point);
$for M in range(0, MR, 2):
__m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point);
@@ -157,8 +169,8 @@
$else:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/MRxNRc4-neondot.c.in b/src/qs8-gemm/MRxNRc4-neondot.c.in
index d395508..eb3af38 100644
--- a/src/qs8-gemm/MRxNRc4-neondot.c.in
+++ b/src/qs8-gemm/MRxNRc4-neondot.c.in
@@ -105,12 +105,12 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
- const int32x4_t vproduct${M}x${ABC[N:N+4]} = vqrdmulhq_n_s32(vacc${M}x${ABC[N:N+4]}, params->neon.multiplier);
+ const int32x4_t vproduct${M}x${ABC[N:N+4]} = vqrdmulhq_n_s32(vacc${M}x${ABC[N:N+4]}, params->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -120,7 +120,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -149,11 +149,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/c16-neon-mlal-padal.c.in b/src/qs8-gemm/c16-neon-mlal-padal.c.in
index 7392300..ebfd029 100644
--- a/src/qs8-gemm/c16-neon-mlal-padal.c.in
+++ b/src/qs8-gemm/c16-neon-mlal-padal.c.in
@@ -104,12 +104,12 @@
int32x4_t vacc${M}x${ABC[N:N+4]} = vcombine_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]} );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -119,7 +119,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -147,11 +147,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/c2-neon-mull-padal-dup.c.in b/src/qs8-gemm/c2-neon-mull-padal-dup.c.in
index 2914567..54e9881 100644
--- a/src/qs8-gemm/c2-neon-mull-padal-dup.c.in
+++ b/src/qs8-gemm/c2-neon-mull-padal-dup.c.in
@@ -139,12 +139,12 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -154,7 +154,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -183,11 +183,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/c8-neon-mull-padal.c.in b/src/qs8-gemm/c8-neon-mull-padal.c.in
index da184fb..e704355 100644
--- a/src/qs8-gemm/c8-neon-mull-padal.c.in
+++ b/src/qs8-gemm/c8-neon-mull-padal.c.in
@@ -122,12 +122,12 @@
int32x4_t vacc${M}x${ABC[N:N+4]} = vcombine_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]} );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -137,7 +137,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -165,11 +165,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 231fdae..2a95279 100644
--- a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -224,13 +224,13 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -242,7 +242,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -254,8 +254,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
index c00e5f7..ab31654 100644
--- a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -222,13 +222,13 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -240,7 +240,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -252,8 +252,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
index b721ca4..9e9ed9c 100644
--- a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -217,13 +217,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -235,7 +235,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -247,8 +247,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
index 3a58012..42751c2 100644
--- a/src/qs8-gemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -176,13 +176,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -194,7 +194,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -205,8 +205,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index f76e520..465927a 100644
--- a/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -242,13 +242,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -260,7 +260,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -272,8 +272,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index df0e6fb..50e5fc7 100644
--- a/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -154,13 +154,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -172,7 +172,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -184,8 +184,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/1x16c4-minmax-gemmlowp-neondot.c
index 1d8d2aa..f0bcc60 100644
--- a/src/qs8-gemm/gen/1x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/1x16c4-minmax-gemmlowp-neondot.c
@@ -97,13 +97,13 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->neon.multiplier);
- const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -115,7 +115,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -127,8 +127,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
index a79b986..9eb5ad8 100644
--- a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
@@ -43,14 +43,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
index ff36aa9..3609135 100644
--- a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -249,13 +249,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -267,7 +267,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -278,8 +278,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
index ed1d96d..e1c439d 100644
--- a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -160,13 +160,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -178,7 +178,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -189,8 +189,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x2-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/1x2-minmax-gemmlowp-scalar.c
index 65fa4d8..7a91fad 100644
--- a/src/qs8-gemm/gen/1x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/1x2-minmax-gemmlowp-scalar.c
@@ -52,7 +52,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
@@ -60,24 +60,24 @@
const int32_t vq31product0x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x0 + vq31rounding) >> 31);
const int32_t vq31product0x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/1x4-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/1x4-minmax-gemmlowp-scalar.c
index dbd2cd8..31866f0 100644
--- a/src/qs8-gemm/gen/1x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/1x4-minmax-gemmlowp-scalar.c
@@ -58,7 +58,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -70,32 +70,32 @@
const int32_t vq31product0x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x2 + vq31rounding) >> 31);
const int32_t vq31product0x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
const int32_t vremainder0x3 = (vq31product0x3 & vremainder_mask) - (int32_t) (vq31product0x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
int32_t vout0x3 = asr_s32(vq31product0x3, vshift) + (int32_t) (vremainder0x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
vout0x3 = math_max_s32(vout0x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
vout0x3 = math_min_s32(vout0x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
index abdb94d..427cb9f 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,23 +119,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
index 4acc0f6..b1c30ad 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,23 +119,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
index 1ae6dca..1836237 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -135,20 +135,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
index ad654f3..4d16fd0 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -135,20 +135,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
index e630e68..e97a400 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,23 +119,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
index 3360b16..278ad8f 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,23 +119,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
index 9eae431..f9cb7c7 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -135,20 +135,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
index 314f342..6e7a449 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -135,20 +135,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
index 5ef3e18..21adf15 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
@@ -110,8 +110,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,23 +124,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
index fec0dce..0b07fc7 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
@@ -110,8 +110,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,23 +124,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
index cfa7d99..3fdd41b 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -112,23 +112,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
index 18399ef..37bf918 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -128,20 +128,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
index 7895ded..ca397ec 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -112,23 +112,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
index 875056a..946b93d 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -128,20 +128,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
index 1b212df..543f507 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
@@ -103,8 +103,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,23 +117,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
index fc4d147..993b2e4 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -91,23 +91,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
index 95e0802..c6f6930 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -93,23 +93,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
index 414f82d..182a674 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -107,20 +107,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
index bd8a13a..3418021 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -109,20 +109,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
index 6aaef3f..8d72bbb 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -91,23 +91,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
index 6ff1e07..e63746f 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -93,23 +93,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
index a885ec0..a54757a 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -107,20 +107,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
index c502eb5..bf5e58a 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -109,20 +109,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index c4af47a..a452a7c 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -91,8 +91,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
@@ -100,22 +100,22 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc00x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc0x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc00x0123, vacc00x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index f97e401..1566b89 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -87,8 +87,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
@@ -96,22 +96,22 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc00x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc0x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc00x0123, vacc00x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
index 26310c8..0809765 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
@@ -82,8 +82,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -96,23 +96,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
index 393e07d..f6399da 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
@@ -84,8 +84,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -98,23 +98,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
index ec0c611..3175475 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -89,23 +89,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
index 863f22e..ddd443e 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -105,20 +105,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
index 23dee4c..1e07d0d 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -89,23 +89,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
index 3ba2f0d..04ec0c9 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -105,20 +105,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-wasmsimd.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-wasmsimd.c
index be88364..a093a7e 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-wasmsimd.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-wasmsimd.c
@@ -87,8 +87,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
@@ -96,22 +96,22 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc00x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc0x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc00x0123, vacc00x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
index 8766948..674b44b 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
@@ -80,8 +80,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -94,23 +94,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 7329a94..3ad4613 100644
--- a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -153,11 +153,11 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -165,7 +165,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -175,8 +175,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
index f3a68a8..62da1a8 100644
--- a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -152,11 +152,11 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -164,7 +164,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -174,8 +174,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
index a5a8fd6..faf07de 100644
--- a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -147,11 +147,11 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -159,7 +159,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -169,8 +169,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 6e110e0..4fbc6cf 100644
--- a/src/qs8-gemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -116,11 +116,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -128,7 +128,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -137,8 +137,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 1944ffe..86a22df 100644
--- a/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -158,11 +158,11 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -170,7 +170,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -180,8 +180,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 822f923..f73c463 100644
--- a/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -110,11 +110,11 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -122,7 +122,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -132,8 +132,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/1x8c4-minmax-gemmlowp-neondot.c
index 3e12c01..bec1770 100644
--- a/src/qs8-gemm/gen/1x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/1x8c4-minmax-gemmlowp-neondot.c
@@ -83,11 +83,11 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -95,7 +95,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -105,8 +105,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..9c1b57c
--- /dev/null
+++ b/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
+
+ vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
index 17dcb23..e1761aa 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
@@ -91,8 +91,8 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
@@ -105,24 +105,24 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
index f8783fb..85b5d1b 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -157,11 +157,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -169,7 +169,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -178,8 +178,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
index 1c010f1..890cecf 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -108,11 +108,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -120,7 +120,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -129,8 +129,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
index 622083f..be4ec37 100644
--- a/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
@@ -87,8 +87,8 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
@@ -101,24 +101,24 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 1c940b0..45cb137 100644
--- a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -298,7 +298,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -308,7 +308,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -328,7 +328,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -346,8 +346,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
index 7e645ff..2b529c9 100644
--- a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -296,7 +296,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -306,7 +306,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -326,7 +326,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -344,8 +344,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
index 4848f85..c51a42b 100644
--- a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -319,7 +319,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -329,7 +329,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -349,7 +349,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -367,8 +367,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
index b7b984a..7409287 100644
--- a/src/qs8-gemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -287,7 +287,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -297,7 +297,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -317,7 +317,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -334,8 +334,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index afae933..209dd15 100644
--- a/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -360,7 +360,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -370,7 +370,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -390,7 +390,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -408,8 +408,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 9e1bac1..34e8361 100644
--- a/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -222,7 +222,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -232,7 +232,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -252,7 +252,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -270,8 +270,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
index 5b4ea8c..447dc07 100644
--- a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
@@ -49,14 +49,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
index e0b9794..716c0c3 100644
--- a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -394,7 +394,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -404,7 +404,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -424,7 +424,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -441,8 +441,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
index 4d865b4..dbdaecc 100644
--- a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -255,7 +255,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -265,7 +265,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -285,7 +285,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -302,8 +302,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x2-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/2x2-minmax-gemmlowp-scalar.c
index 93d7387..9f1c433 100644
--- a/src/qs8-gemm/gen/2x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/2x2-minmax-gemmlowp-scalar.c
@@ -63,7 +63,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -75,32 +75,32 @@
const int32_t vq31product1x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x0 + vq31rounding) >> 31);
const int32_t vq31product1x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
const int32_t vremainder1x1 = (vq31product1x1 & vremainder_mask) - (int32_t) (vq31product1x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
int32_t vout1x1 = asr_s32(vq31product1x1, vshift) + (int32_t) (vremainder1x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
vout1x1 = math_max_s32(vout1x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
vout1x1 = math_min_s32(vout1x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/2x4-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/2x4-minmax-gemmlowp-scalar.c
index 7f67584..29d8508 100644
--- a/src/qs8-gemm/gen/2x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/2x4-minmax-gemmlowp-scalar.c
@@ -73,7 +73,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -93,7 +93,7 @@
const int32_t vq31product1x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x2 + vq31rounding) >> 31);
const int32_t vq31product1x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -103,8 +103,8 @@
const int32_t vremainder1x2 = (vq31product1x2 & vremainder_mask) - (int32_t) (vq31product1x2 < 0);
const int32_t vremainder1x3 = (vq31product1x3 & vremainder_mask) - (int32_t) (vq31product1x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -114,7 +114,7 @@
int32_t vout1x2 = asr_s32(vq31product1x2, vshift) + (int32_t) (vremainder1x2 > vremainder_threshold);
int32_t vout1x3 = asr_s32(vq31product1x3, vshift) + (int32_t) (vremainder1x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -124,7 +124,7 @@
vout1x2 = math_max_s32(vout1x2, vout_min);
vout1x3 = math_max_s32(vout1x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -134,7 +134,7 @@
vout1x2 = math_min_s32(vout1x2, vout_max);
vout1x3 = math_min_s32(vout1x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
index 5c8c65f..de0580f 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,27 +152,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
index 73d5422..4213ba5 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,27 +152,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
index b5be8c7..3314380 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -176,24 +176,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
index 453aceb..3e0a980 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -176,24 +176,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
index 1c31e16..e59f622 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,27 +152,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
index b75bcde..5bbf9c3 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,27 +152,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
index 3b732b8..a1eb413 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -176,24 +176,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
index f177ef9..d917ca9 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -176,24 +176,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
index 6f3dae5..42fec23 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
@@ -137,8 +137,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -157,27 +157,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
index 04f48d1..f225098 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
@@ -137,8 +137,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -157,27 +157,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
index 6d70934..1f998e1 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,27 +145,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
index f040a7b..ea6a894 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -169,24 +169,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
index 6d6fd5a..ff5840a 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,27 +145,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
index 8f67268..895b0cf 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -169,24 +169,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
index cf0622b..4bc8c55 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
@@ -130,8 +130,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -150,27 +150,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
index 26c20db..df5de10 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,27 +117,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
index 9549f20..ae85181 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,27 +119,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
index 6afe7f8..85f2c10 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -141,24 +141,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
index aa2c3f8..9813058 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -143,24 +143,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
index 063b30a..1691be1 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,27 +117,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
index 086af13..c619ae7 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,27 +119,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
index 663f448..bd7bd88 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -141,24 +141,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
index 8e5eca9..2928523 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -143,24 +143,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index eccd1b3..085e9a9 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -120,8 +120,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -133,24 +133,24 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc01x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index 35141ec..83fdd00 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -116,8 +116,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -129,24 +129,24 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc01x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
index d1d91f5..6903e30 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
@@ -102,8 +102,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -122,27 +122,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
index 42f3678..c38e481 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
@@ -104,8 +104,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,27 +124,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
index f4f8c0b..86f9c1d 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -115,27 +115,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
index 344df2a..8b67fcd 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -139,24 +139,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
index 5afe66e..ea4d124 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -115,27 +115,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
index 5985c84..54d9ab4 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -139,24 +139,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-wasmsimd.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-wasmsimd.c
index 02ed0f8..50d00a3 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-wasmsimd.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-wasmsimd.c
@@ -116,8 +116,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -129,24 +129,24 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc01x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
index 6044996..a22b257 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
@@ -100,8 +100,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -120,27 +120,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 87f0291..8208fa0 100644
--- a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -195,13 +195,13 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -213,7 +213,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -225,8 +225,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
index 687a9b3..c5e0cdb 100644
--- a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -194,13 +194,13 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -212,7 +212,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -224,8 +224,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
index de05fc5..1df366c 100644
--- a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -202,13 +202,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -220,7 +220,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -232,8 +232,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 64eafde..9e6ecc3 100644
--- a/src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -175,13 +175,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -193,7 +193,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -204,8 +204,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 8531b93..0572c92 100644
--- a/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -222,13 +222,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -240,7 +240,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -252,8 +252,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index de48d33..9741b73 100644
--- a/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -148,13 +148,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -166,7 +166,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -178,8 +178,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..d63d4f9
--- /dev/null
+++ b/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,178 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+ _mm_storeu_si32(c1, vout_hi);
+
+ c0 += 4;
+ c1 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+
+ c0 += 2;
+ c1 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
index d388284..1e39921 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
@@ -112,8 +112,8 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
@@ -132,28 +132,28 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 923d485..6d45009 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -234,13 +234,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -252,7 +252,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -263,8 +263,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
index e05b02b..612c0bc 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -159,13 +159,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -177,7 +177,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -188,8 +188,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
index 6c39642..810bae4 100644
--- a/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
@@ -108,8 +108,8 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
@@ -128,28 +128,28 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 0c56b19..7729647 100644
--- a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -372,7 +372,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -386,7 +386,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -414,7 +414,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -438,8 +438,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
index 58deb96..b95411bf 100644
--- a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -370,7 +370,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -384,7 +384,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -412,7 +412,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -436,8 +436,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
index b17e441..55436f4 100644
--- a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -421,7 +421,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -435,7 +435,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -463,7 +463,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -487,8 +487,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
index e57ca14..3063ac0 100644
--- a/src/qs8-gemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -398,7 +398,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -412,7 +412,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -440,7 +440,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -463,8 +463,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index ce9b048..dca07ae 100644
--- a/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -478,7 +478,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -492,7 +492,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -520,7 +520,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -544,8 +544,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index b021ba0..01be9bb 100644
--- a/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -290,7 +290,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -304,7 +304,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -332,7 +332,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -356,8 +356,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
index e94d74c..be2d1b5 100644
--- a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
@@ -55,14 +55,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
index 60be3af..b9aa38f 100644
--- a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -539,7 +539,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -553,7 +553,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -581,7 +581,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -604,8 +604,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
index ca496b1..d328f83 100644
--- a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -350,7 +350,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -364,7 +364,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -392,7 +392,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -415,8 +415,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x2-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/3x2-minmax-gemmlowp-scalar.c
index 10a266a..5b9009b 100644
--- a/src/qs8-gemm/gen/3x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/3x2-minmax-gemmlowp-scalar.c
@@ -74,7 +74,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -90,7 +90,7 @@
const int32_t vq31product2x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x0 + vq31rounding) >> 31);
const int32_t vq31product2x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
@@ -98,8 +98,8 @@
const int32_t vremainder2x0 = (vq31product2x0 & vremainder_mask) - (int32_t) (vq31product2x0 < 0);
const int32_t vremainder2x1 = (vq31product2x1 & vremainder_mask) - (int32_t) (vq31product2x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
@@ -107,7 +107,7 @@
int32_t vout2x0 = asr_s32(vq31product2x0, vshift) + (int32_t) (vremainder2x0 > vremainder_threshold);
int32_t vout2x1 = asr_s32(vq31product2x1, vshift) + (int32_t) (vremainder2x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
@@ -115,7 +115,7 @@
vout2x0 = math_max_s32(vout2x0, vout_min);
vout2x1 = math_max_s32(vout2x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
@@ -123,7 +123,7 @@
vout2x0 = math_min_s32(vout2x0, vout_max);
vout2x1 = math_min_s32(vout2x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/3x4-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/3x4-minmax-gemmlowp-scalar.c
index 61c64a9..1ae2a6b 100644
--- a/src/qs8-gemm/gen/3x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/3x4-minmax-gemmlowp-scalar.c
@@ -88,7 +88,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -116,7 +116,7 @@
const int32_t vq31product2x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x2 + vq31rounding) >> 31);
const int32_t vq31product2x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -130,8 +130,8 @@
const int32_t vremainder2x2 = (vq31product2x2 & vremainder_mask) - (int32_t) (vq31product2x2 < 0);
const int32_t vremainder2x3 = (vq31product2x3 & vremainder_mask) - (int32_t) (vq31product2x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -145,7 +145,7 @@
int32_t vout2x2 = asr_s32(vq31product2x2, vshift) + (int32_t) (vremainder2x2 > vremainder_threshold);
int32_t vout2x3 = asr_s32(vq31product2x3, vshift) + (int32_t) (vremainder2x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -159,7 +159,7 @@
vout2x2 = math_max_s32(vout2x2, vout_min);
vout2x3 = math_max_s32(vout2x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -173,7 +173,7 @@
vout2x2 = math_min_s32(vout2x2, vout_max);
vout2x3 = math_min_s32(vout2x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
index 885766e..acf1eb3 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,15 +202,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
index 99e227f..1d059b7 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,15 +202,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
index 56635b9..8cc7035 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -217,7 +217,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -225,8 +225,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -234,12 +234,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
index 39e555d..a01a80a 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -217,7 +217,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -225,8 +225,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -234,12 +234,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
index e106ae8..a6a16fa 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,15 +202,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
index 18c00d6..c60f805 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,15 +202,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
index c89d3bc..b693059 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -217,7 +217,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -225,8 +225,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -234,12 +234,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
index d033069..7193059 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -217,7 +217,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -225,8 +225,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -234,12 +234,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
index 49e37b1..525d542 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
@@ -164,8 +164,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -190,7 +190,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -198,8 +198,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -207,15 +207,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
index bb6ea3a..9fb43b5 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
@@ -164,8 +164,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -190,7 +190,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -198,8 +198,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -207,15 +207,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
index aa07d11..b58ef12 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -178,7 +178,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -186,8 +186,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -195,15 +195,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
index e1535ac..e43eb3e 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -210,7 +210,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -218,8 +218,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -227,12 +227,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
index c481125..22dd63a 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -178,7 +178,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -186,8 +186,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -195,15 +195,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
index 5f8686a..cb0a4f4 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -210,7 +210,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -218,8 +218,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -227,12 +227,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
index 609ec98..b0554bc 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
@@ -157,8 +157,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -183,7 +183,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -191,8 +191,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -200,15 +200,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
index 42aeaaa..2d31d96 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -143,7 +143,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -151,8 +151,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -160,15 +160,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
index 367ed89..bc3e045 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,7 +145,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -153,8 +153,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -162,15 +162,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
index 156d22e..083b15f 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
__m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -175,7 +175,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -183,8 +183,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -192,12 +192,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
index 5e401f5..f470c48 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
__m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -177,7 +177,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -185,8 +185,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -194,12 +194,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
index d55b408..c8d66cb 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -143,7 +143,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -151,8 +151,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -160,15 +160,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
index b7e914a..57b96a5 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,7 +145,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -153,8 +153,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -162,15 +162,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
index fbd11c4..148842d 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -175,7 +175,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -183,8 +183,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -192,12 +192,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
index e64c763..6eb9941 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -177,7 +177,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -185,8 +185,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -194,12 +194,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index 02387e4..a37bb42 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -149,8 +149,8 @@
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
const v128_t vacc2x01 = wasm_v32x4_shuffle(vacc2x0123, vsign2x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -166,27 +166,27 @@
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
const v128_t vq31prod2x0123 = wasm_v32x4_shuffle(vprod2x01, vprod2x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
const v128_t vrem2x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod2x0123, vremainder_mask), wasm_i32x4_lt(vq31prod2x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
vacc2x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod2x0123, vshift), wasm_i32x4_gt(vrem2x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vacc22x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc2x0123, vacc2x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc22x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index 150bbfb..6ed27ee 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -145,8 +145,8 @@
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
const v128_t vacc2x01 = wasm_v32x4_shuffle(vacc2x0123, vsign2x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -162,27 +162,27 @@
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
const v128_t vq31prod2x0123 = wasm_v32x4_shuffle(vprod2x01, vprod2x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
const v128_t vrem2x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod2x0123, vremainder_mask), wasm_i32x4_lt(vq31prod2x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
vacc2x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod2x0123, vshift), wasm_i32x4_gt(vrem2x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vacc22x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc2x0123, vacc2x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc22x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
index 7bea2fd..c8245f1 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
@@ -122,8 +122,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -148,7 +148,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -156,8 +156,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -165,15 +165,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
index 5bafe10..560ba40 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
@@ -124,8 +124,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -150,7 +150,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -158,8 +158,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -167,15 +167,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
index 544b4e2..3441a20 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -141,7 +141,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -149,8 +149,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -158,15 +158,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
index d4fb08a..9178d3e 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
__m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -173,7 +173,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -181,8 +181,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -190,12 +190,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
index b9df585..1e5ef35 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -141,7 +141,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -149,8 +149,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -158,15 +158,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
index fd41846..65119a7 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -173,7 +173,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -181,8 +181,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -190,12 +190,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-wasmsimd.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-wasmsimd.c
index 5b8c51e..adb57cf 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-wasmsimd.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-wasmsimd.c
@@ -145,8 +145,8 @@
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
const v128_t vacc2x01 = wasm_v32x4_shuffle(vacc2x0123, vsign2x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -162,27 +162,27 @@
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
const v128_t vq31prod2x0123 = wasm_v32x4_shuffle(vprod2x01, vprod2x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
const v128_t vrem2x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod2x0123, vremainder_mask), wasm_i32x4_lt(vq31prod2x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
vacc2x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod2x0123, vshift), wasm_i32x4_gt(vrem2x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vacc22x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc2x0123, vacc2x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc22x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
index 995ea8b..7a091ab 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
@@ -120,8 +120,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -146,7 +146,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -154,8 +154,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -163,15 +163,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 6126c00..b8f255c 100644
--- a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -237,7 +237,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -245,7 +245,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -261,7 +261,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -277,8 +277,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
index cd20044..9ec08f6 100644
--- a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -236,7 +236,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -244,7 +244,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -260,7 +260,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -276,8 +276,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
index aa3397d..f6608d9 100644
--- a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -257,7 +257,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -265,7 +265,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -281,7 +281,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -297,8 +297,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 3dd59a6..6c73b05 100644
--- a/src/qs8-gemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -234,7 +234,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -242,7 +242,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -258,7 +258,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -273,8 +273,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index e3abc2a..ee3d8fa 100644
--- a/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -286,7 +286,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -294,7 +294,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -310,7 +310,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -326,8 +326,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 428330a..7a1ffad 100644
--- a/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -186,7 +186,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -194,7 +194,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -210,7 +210,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -226,8 +226,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..60b38a7
--- /dev/null
+++ b/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,212 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ __m256i vacc2x01 = vacc0x01;
+ __m256i vacc2x23 = vacc0x23;
+ __m256i vacc2x45 = vacc0x45;
+ __m256i vacc2x67 = vacc0x67;
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a2));
+ const __m256i vxa2 = _mm256_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+ vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+ const __m256i vacc2x0213 = _mm256_hadd_epi32(vacc2x01, vacc2x23);
+ const __m256i vacc2x4657 = _mm256_hadd_epi32(vacc2x45, vacc2x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+ const __m256i vacc2x02461357 = _mm256_hadd_epi32(vacc2x0213, vacc2x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+ __m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+ __m256 vscaled2x01234567 = _mm256_cvtepi32_ps(vacc2x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+ vscaled2x01234567 = _mm256_mul_ps(vscaled2x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+ vacc2x01234567 = _mm256_cvtps_epi32(vscaled2x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+ __m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+ vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+ _mm_storeh_pi((__m64*) c2, _mm_castsi128_ps(vout_lo));
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+ _mm_storeu_si32(c1, vout_hi);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout_lo, 2);
+
+ c0 += 4;
+ c1 += 4;
+ c2 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout_lo, 4);
+
+ c0 += 2;
+ c1 += 2;
+ c2 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
index 68c2e34..b87cc62 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
@@ -133,8 +133,8 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
@@ -159,7 +159,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -167,8 +167,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -176,7 +176,7 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
@@ -185,8 +185,8 @@
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 564e036..85349cb 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -311,7 +311,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -319,7 +319,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -335,7 +335,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -350,8 +350,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
index 38f9282..deb14a7 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -210,7 +210,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -218,7 +218,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -234,7 +234,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -249,8 +249,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
index 6772f39..0a4326c 100644
--- a/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
@@ -129,8 +129,8 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
@@ -155,7 +155,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -163,8 +163,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -172,7 +172,7 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
@@ -181,8 +181,8 @@
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 249c1a7..5ee36f5 100644
--- a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -446,7 +446,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -464,7 +464,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -500,7 +500,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -530,8 +530,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
index 90c23bc..0e78e76 100644
--- a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -444,7 +444,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -462,7 +462,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -498,7 +498,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -528,8 +528,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
index 000c5eb..a0bba2b 100644
--- a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -523,7 +523,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -541,7 +541,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -577,7 +577,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -607,8 +607,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
index 70da22e..1a7fde3 100644
--- a/src/qs8-gemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -509,7 +509,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -527,7 +527,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -563,7 +563,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -592,8 +592,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 1f1884f..71a2a88 100644
--- a/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -596,7 +596,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -614,7 +614,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -650,7 +650,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -680,8 +680,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 746bcb3..ed74770 100644
--- a/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -358,7 +358,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -376,7 +376,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -412,7 +412,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -442,8 +442,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/4x16c4-minmax-gemmlowp-neondot.c
index c21610b..6b931a0 100644
--- a/src/qs8-gemm/gen/4x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/4x16c4-minmax-gemmlowp-neondot.c
@@ -169,25 +169,25 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->neon.multiplier);
- const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->neon.multiplier);
- const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->neon.multiplier);
- const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->neon.multiplier);
- const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -223,7 +223,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -253,8 +253,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
index 5439a53..8ff8bbe 100644
--- a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
@@ -61,14 +61,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
index f03e0e8..7ce34e0 100644
--- a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -684,7 +684,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -702,7 +702,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -738,7 +738,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -767,8 +767,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
index c5dea1e..2fb0196 100644
--- a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -445,7 +445,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -463,7 +463,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -499,7 +499,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -528,8 +528,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x2-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/4x2-minmax-gemmlowp-scalar.c
index 9679d29..608db29 100644
--- a/src/qs8-gemm/gen/4x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/4x2-minmax-gemmlowp-scalar.c
@@ -85,7 +85,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -105,7 +105,7 @@
const int32_t vq31product3x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x0 + vq31rounding) >> 31);
const int32_t vq31product3x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
@@ -115,8 +115,8 @@
const int32_t vremainder3x0 = (vq31product3x0 & vremainder_mask) - (int32_t) (vq31product3x0 < 0);
const int32_t vremainder3x1 = (vq31product3x1 & vremainder_mask) - (int32_t) (vq31product3x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
@@ -126,7 +126,7 @@
int32_t vout3x0 = asr_s32(vq31product3x0, vshift) + (int32_t) (vremainder3x0 > vremainder_threshold);
int32_t vout3x1 = asr_s32(vq31product3x1, vshift) + (int32_t) (vremainder3x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
@@ -136,7 +136,7 @@
vout3x0 = math_max_s32(vout3x0, vout_min);
vout3x1 = math_max_s32(vout3x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
@@ -146,7 +146,7 @@
vout3x0 = math_min_s32(vout3x0, vout_max);
vout3x1 = math_min_s32(vout3x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/4x4-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/4x4-minmax-gemmlowp-scalar.c
index 16419b8..74c469f 100644
--- a/src/qs8-gemm/gen/4x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/4x4-minmax-gemmlowp-scalar.c
@@ -103,7 +103,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -139,7 +139,7 @@
const int32_t vq31product3x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x2 + vq31rounding) >> 31);
const int32_t vq31product3x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -157,8 +157,8 @@
const int32_t vremainder3x2 = (vq31product3x2 & vremainder_mask) - (int32_t) (vq31product3x2 < 0);
const int32_t vremainder3x3 = (vq31product3x3 & vremainder_mask) - (int32_t) (vq31product3x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -176,7 +176,7 @@
int32_t vout3x2 = asr_s32(vq31product3x2, vshift) + (int32_t) (vremainder3x2 > vremainder_threshold);
int32_t vout3x3 = asr_s32(vq31product3x3, vshift) + (int32_t) (vremainder3x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -194,7 +194,7 @@
vout3x2 = math_max_s32(vout3x2, vout_min);
vout3x3 = math_max_s32(vout3x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -212,7 +212,7 @@
vout3x2 = math_min_s32(vout3x2, vout_max);
vout3x3 = math_min_s32(vout3x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
index b0cec37..6ee3c62 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,15 +239,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
index a3db995..939d732 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,15 +239,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
index 17f7643..df5ede3 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -258,7 +258,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -268,8 +268,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -279,12 +279,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
index d2f821c..9882976 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -258,7 +258,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -268,8 +268,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -279,12 +279,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
index 2d53868..a7ea9d9 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,15 +239,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
index dafa8dc..64fb13f 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,15 +239,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
index 9c8cf70..739e36b 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -258,7 +258,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -268,8 +268,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -279,12 +279,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
index 1225935..fb97abd 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -258,7 +258,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -268,8 +268,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -279,12 +279,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
index 10bf278..4a2cf7b 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
@@ -191,8 +191,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -223,7 +223,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -233,8 +233,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -244,15 +244,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
index 4e2b48a..0af6be9 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
@@ -191,8 +191,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -223,7 +223,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -233,8 +233,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -244,15 +244,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
index 06a057a..de0f304 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -211,7 +211,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -221,8 +221,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -232,15 +232,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
index 127c98b..ed56800 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -251,7 +251,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -261,8 +261,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -272,12 +272,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
index 93f66b1..29df1a0 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -211,7 +211,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -221,8 +221,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -232,15 +232,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
index 357bec9..c52fd01 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -251,7 +251,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -261,8 +261,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -272,12 +272,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
index 61952bd..b4a5242 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
@@ -184,8 +184,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -216,7 +216,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -226,8 +226,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -237,15 +237,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 36b85b2..9b6ed23 100644
--- a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -279,7 +279,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -289,7 +289,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -309,7 +309,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -327,8 +327,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
index fa00c3a..de22292 100644
--- a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -278,7 +278,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -288,7 +288,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -308,7 +308,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -326,8 +326,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
index c53e78c..2c78c76 100644
--- a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -312,7 +312,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -322,7 +322,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -342,7 +342,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -360,8 +360,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 4351ac3..1ee332c 100644
--- a/src/qs8-gemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -293,7 +293,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -303,7 +303,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -323,7 +323,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -340,8 +340,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index acf3d2e..42b9632 100644
--- a/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -350,7 +350,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -360,7 +360,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -380,7 +380,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -398,8 +398,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index d640aef..c7acc44 100644
--- a/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -224,7 +224,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -234,7 +234,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -254,7 +254,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -272,8 +272,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/4x8c4-minmax-gemmlowp-neondot.c
index 949f81b..266a756 100644
--- a/src/qs8-gemm/gen/4x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/4x8c4-minmax-gemmlowp-neondot.c
@@ -131,17 +131,17 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -161,7 +161,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -179,8 +179,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 5d4c337..b4ad38c 100644
--- a/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -388,7 +388,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -398,7 +398,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -418,7 +418,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -435,8 +435,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
index 81a0a81..7042d13 100644
--- a/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -261,7 +261,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -271,7 +271,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -291,7 +291,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -308,8 +308,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 3a9b99b..5fd4633 100644
--- a/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -594,7 +594,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -620,7 +620,7 @@
vacc5x89AB = vqrdmulhq_s32(vacc5x89AB, vmultiplier);
vacc5xCDEF = vqrdmulhq_s32(vacc5xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -672,7 +672,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -714,8 +714,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
index b7bb61b..2812abd 100644
--- a/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -592,7 +592,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -618,7 +618,7 @@
vacc5x89AB = vqrdmulhq_s32(vacc5x89AB, vmultiplier);
vacc5xCDEF = vqrdmulhq_s32(vacc5xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -670,7 +670,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -712,8 +712,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/6x16c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/6x16c4-minmax-gemmlowp-neondot.c
index 8eb91a7..abf5913 100644
--- a/src/qs8-gemm/gen/6x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/6x16c4-minmax-gemmlowp-neondot.c
@@ -217,33 +217,33 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->neon.multiplier);
- const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->neon.multiplier);
- const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->neon.multiplier);
- const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->neon.multiplier);
- const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->neon.multiplier);
- const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier);
- const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier);
- const int32x4_t vproduct4x89AB = vqrdmulhq_n_s32(vacc4x89AB, params->neon.multiplier);
- const int32x4_t vproduct4xCDEF = vqrdmulhq_n_s32(vacc4xCDEF, params->neon.multiplier);
- const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier);
- const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier);
- const int32x4_t vproduct5x89AB = vqrdmulhq_n_s32(vacc5x89AB, params->neon.multiplier);
- const int32x4_t vproduct5xCDEF = vqrdmulhq_n_s32(vacc5xCDEF, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x89AB = vqrdmulhq_n_s32(vacc4x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4xCDEF = vqrdmulhq_n_s32(vacc4xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x89AB = vqrdmulhq_n_s32(vacc5x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5xCDEF = vqrdmulhq_n_s32(vacc5xCDEF, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -295,7 +295,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -337,8 +337,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 2bb0e53..b08dfd9 100644
--- a/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -363,7 +363,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -377,7 +377,7 @@
vacc5x0123 = vqrdmulhq_s32(vacc5x0123, vmultiplier);
vacc5x4567 = vqrdmulhq_s32(vacc5x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -405,7 +405,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -429,8 +429,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
index d36d452..2336529 100644
--- a/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -362,7 +362,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -376,7 +376,7 @@
vacc5x0123 = vqrdmulhq_s32(vacc5x0123, vmultiplier);
vacc5x4567 = vqrdmulhq_s32(vacc5x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -404,7 +404,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -428,8 +428,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/6x8c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/6x8c4-minmax-gemmlowp-neondot.c
index 230bbff..6a54838 100644
--- a/src/qs8-gemm/gen/6x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/6x8c4-minmax-gemmlowp-neondot.c
@@ -163,21 +163,21 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier);
- const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier);
- const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier);
- const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -205,7 +205,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -229,8 +229,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/8x16c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/8x16c4-minmax-gemmlowp-neondot.c
index da350ec..78f241d 100644
--- a/src/qs8-gemm/gen/8x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/8x16c4-minmax-gemmlowp-neondot.c
@@ -265,41 +265,41 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->neon.multiplier);
- const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->neon.multiplier);
- const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->neon.multiplier);
- const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->neon.multiplier);
- const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->neon.multiplier);
- const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier);
- const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier);
- const int32x4_t vproduct4x89AB = vqrdmulhq_n_s32(vacc4x89AB, params->neon.multiplier);
- const int32x4_t vproduct4xCDEF = vqrdmulhq_n_s32(vacc4xCDEF, params->neon.multiplier);
- const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier);
- const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier);
- const int32x4_t vproduct5x89AB = vqrdmulhq_n_s32(vacc5x89AB, params->neon.multiplier);
- const int32x4_t vproduct5xCDEF = vqrdmulhq_n_s32(vacc5xCDEF, params->neon.multiplier);
- const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->neon.multiplier);
- const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->neon.multiplier);
- const int32x4_t vproduct6x89AB = vqrdmulhq_n_s32(vacc6x89AB, params->neon.multiplier);
- const int32x4_t vproduct6xCDEF = vqrdmulhq_n_s32(vacc6xCDEF, params->neon.multiplier);
- const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->neon.multiplier);
- const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->neon.multiplier);
- const int32x4_t vproduct7x89AB = vqrdmulhq_n_s32(vacc7x89AB, params->neon.multiplier);
- const int32x4_t vproduct7xCDEF = vqrdmulhq_n_s32(vacc7xCDEF, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x89AB = vqrdmulhq_n_s32(vacc4x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4xCDEF = vqrdmulhq_n_s32(vacc4xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x89AB = vqrdmulhq_n_s32(vacc5x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5xCDEF = vqrdmulhq_n_s32(vacc5xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x89AB = vqrdmulhq_n_s32(vacc6x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6xCDEF = vqrdmulhq_n_s32(vacc6xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x89AB = vqrdmulhq_n_s32(vacc7x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7xCDEF = vqrdmulhq_n_s32(vacc7xCDEF, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -367,7 +367,7 @@
vacc7x89AB = vrshlq_s32(vacc7x89AB, vright_shift);
vacc7xCDEF = vrshlq_s32(vacc7xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -421,8 +421,8 @@
int8x16_t vout6x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc6x89ABCDEF));
int8x16_t vout7x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc7x01234567), vqmovn_s16(vacc7x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/8x8c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/8x8c4-minmax-gemmlowp-neondot.c
index 1c9501d..a01922e 100644
--- a/src/qs8-gemm/gen/8x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/8x8c4-minmax-gemmlowp-neondot.c
@@ -195,25 +195,25 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier);
- const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier);
- const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier);
- const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier);
- const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->neon.multiplier);
- const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->neon.multiplier);
- const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->neon.multiplier);
- const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -249,7 +249,7 @@
vacc7x0123 = vrshlq_s32(vacc7x0123, vright_shift);
vacc7x4567 = vrshlq_s32(vacc7x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -279,8 +279,8 @@
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
int8x16_t vout6x01234567_7x01234567 = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc7x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/neon-mlal-lane.c.in b/src/qs8-gemm/neon-mlal-lane.c.in
index 6db2cae..e1a2829 100644
--- a/src/qs8-gemm/neon-mlal-lane.c.in
+++ b/src/qs8-gemm/neon-mlal-lane.c.in
@@ -174,12 +174,12 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -189,7 +189,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -218,11 +218,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/neon-mull-addw-dup.c.in b/src/qs8-gemm/neon-mull-addw-dup.c.in
index 42e01dd..8cbf8aa 100644
--- a/src/qs8-gemm/neon-mull-addw-dup.c.in
+++ b/src/qs8-gemm/neon-mull-addw-dup.c.in
@@ -158,12 +158,12 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -173,7 +173,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -202,11 +202,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/scalar.c.in b/src/qs8-gemm/scalar.c.in
index 4b813a2..4d8f133 100644
--- a/src/qs8-gemm/scalar.c.in
+++ b/src/qs8-gemm/scalar.c.in
@@ -71,7 +71,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
$for M in range(MR):
$for N in range(NR):
const int64_t vproduct${M}x${N} = (int64_t) vacc${M}x${N} * (int64_t) vmultiplier;
@@ -81,28 +81,28 @@
$for N in range(NR):
const int32_t vq31product${M}x${N} = (int32_t) (uint32_t) ((uint64_t) (vproduct${M}x${N} + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
$for M in range(MR):
$for N in range(NR):
const int32_t vremainder${M}x${N} = (vq31product${M}x${N} & vremainder_mask) - (int32_t) (vq31product${M}x${N} < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
$for M in range(MR):
$for N in range(NR):
int32_t vout${M}x${N} = asr_s32(vq31product${M}x${N}, vshift) + (int32_t) (vremainder${M}x${N} > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} = math_max_s32(vout${M}x${N}, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} = math_min_s32(vout${M}x${N}, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} += voutput_zero_point;
diff --git a/src/qs8-igemm/MRx16c8-avx512skx.c.in b/src/qs8-igemm/MRx16c8-avx512skx.c.in
index 5896443..1ac67b4 100644
--- a/src/qs8-igemm/MRx16c8-avx512skx.c.in
+++ b/src/qs8-igemm/MRx16c8-avx512skx.c.in
@@ -58,24 +58,24 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
$if MR > 1:
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
$else:
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
$if MR > 2:
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
$elif MR == 2:
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
$else:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
$for N in range(4, 16, 4):
diff --git a/src/qs8-igemm/MRx4c2-sse.c.in b/src/qs8-igemm/MRx4c2-sse.c.in
index 1d58a24..d08425d 100644
--- a/src/qs8-igemm/MRx4c2-sse.c.in
+++ b/src/qs8-igemm/MRx4c2-sse.c.in
@@ -25,7 +25,7 @@
#include <xnnpack/math.h>
-$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
+$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_igemm_minmax_gemmlowp_ukernel_${MR}x4c2__${ISA}_${VARIANT.lower()}(
size_t mr,
@@ -275,8 +275,8 @@
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -286,8 +286,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
$for M in reversed(range(1, MR)):
diff --git a/src/qs8-igemm/MRx4c8-sse.c.in b/src/qs8-igemm/MRx4c8-sse.c.in
index ef7ac64..b616122 100644
--- a/src/qs8-igemm/MRx4c8-sse.c.in
+++ b/src/qs8-igemm/MRx4c8-sse.c.in
@@ -25,7 +25,7 @@
#include <xnnpack/math.h>
-$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
+$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_igemm_minmax_gemmlowp_ukernel_${MR}x4c8__${ISA}_${VARIANT.lower()}(
size_t mr,
@@ -233,8 +233,8 @@
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -244,8 +244,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
$for M in reversed(range(1, MR)):
diff --git a/src/qs8-igemm/MRx4c8-wasmsimd.c.in b/src/qs8-igemm/MRx4c8-wasmsimd.c.in
index c55267f..014dadd 100644
--- a/src/qs8-igemm/MRx4c8-wasmsimd.c.in
+++ b/src/qs8-igemm/MRx4c8-wasmsimd.c.in
@@ -141,8 +141,8 @@
$for M in range(MR):
const v128_t vacc${M}x01 = wasm_v32x4_shuffle(vacc${M}x0123, vsign${M}x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
$for M in range(MR):
const v128_t vprod${M}x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc${M}x01, vmultiplier), vrounding);
const v128_t vacc${M}x23 = wasm_v32x4_shuffle(vacc${M}x0123, vsign${M}x0123, 2, 6, 3, 7);
@@ -153,16 +153,16 @@
$for M in range(MR):
const v128_t vq31prod${M}x0123 = wasm_v32x4_shuffle(vprod${M}x01, vprod${M}x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
$for M in range(MR):
const v128_t vrem${M}x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod${M}x0123, vremainder_mask), wasm_i32x4_lt(vq31prod${M}x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
$for M in range(MR):
vacc${M}x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${M}x0123, vshift), wasm_i32x4_gt(vrem${M}x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
$for M in range(0, MR, 2):
v128_t vacc${M}${min(M+1, MR-1)}x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
@@ -171,10 +171,10 @@
$else:
v128_t vout = wasm_i8x16_narrow_i16x8(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-igemm/MRx8c8-avx2.c.in b/src/qs8-igemm/MRx8c8-avx2.c.in
index a78ef8a..f18999e 100644
--- a/src/qs8-igemm/MRx8c8-avx2.c.in
+++ b/src/qs8-igemm/MRx8c8-avx2.c.in
@@ -3,6 +3,7 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert MR <= 4
#include <assert.h>
@@ -13,7 +14,7 @@
#include <xnnpack/math.h>
-void xnn_qs8_igemm_minmax_gemmlowp_ukernel_${MR}x8c8__avx2(
+void xnn_qs8_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x8c8__avx2(
size_t mr,
size_t nc,
size_t kc,
@@ -111,40 +112,51 @@
$for M in range(MR):
__m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
- $for M in range(MR):
- const __m256i vacc${M}x11335577 = _mm256_shuffle_epi32(vacc${M}x01234567, _MM_SHUFFLE(3, 3, 1, 1));
+ $for M in range(MR):
+ const __m256i vacc${M}x11335577 = _mm256_shuffle_epi32(vacc${M}x01234567, _MM_SHUFFLE(3, 3, 1, 1));
- $for M in range(MR):
- const __m256i vprod${M}x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x01234567, vmultiplier), vrounding);
+ $for M in range(MR):
+ const __m256i vprod${M}x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x01234567, vmultiplier), vrounding);
- $for M in range(MR):
- const __m256i vprod${M}x1357 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x11335577, vmultiplier), vrounding);
+ $for M in range(MR):
+ const __m256i vprod${M}x1357 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x11335577, vmultiplier), vrounding);
- $for M in range(MR):
- const __m256i vq31prod${M}x0246 = _mm256_srli_epi64(vprod${M}x0246, 31);
- const __m256i vq31prod${M}x1357 = _mm256_add_epi64(vprod${M}x1357, vprod${M}x1357);
+ $for M in range(MR):
+ const __m256i vq31prod${M}x0246 = _mm256_srli_epi64(vprod${M}x0246, 31);
+ const __m256i vq31prod${M}x1357 = _mm256_add_epi64(vprod${M}x1357, vprod${M}x1357);
- $for M in range(MR):
- const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
+ $for M in range(MR):
+ const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
- $for M in range(MR):
- const __m256i vrem${M}x01234567 =
- _mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
+ $for M in range(MR):
+ const __m256i vrem${M}x01234567 =
+ _mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
+ $for M in range(MR):
+ vacc${M}x01234567 =
+ _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
$else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
- $for M in range(MR):
- vacc${M}x01234567 =
- _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
+ $for M in range(MR):
+ __m256 vscaled${M}x01234567 = _mm256_cvtepi32_ps(vacc${M}x01234567);
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ $for M in range(MR):
+ vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale);
+
+ $for M in range(MR):
+ vacc${M}x01234567 = _mm256_cvtps_epi32(vscaled${M}x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_zero_point);
$for M in range(0, MR, 2):
__m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point);
@@ -156,8 +168,8 @@
$else:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/MRxNRc4-neondot.c.in b/src/qs8-igemm/MRxNRc4-neondot.c.in
index 01b243d..bce09a9 100644
--- a/src/qs8-igemm/MRxNRc4-neondot.c.in
+++ b/src/qs8-igemm/MRxNRc4-neondot.c.in
@@ -111,12 +111,12 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -126,7 +126,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -155,11 +155,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in reversed(range(MR)):
$for N in range(0, NR, 16):
diff --git a/src/qs8-igemm/c16-neon-mlal-padal.c.in b/src/qs8-igemm/c16-neon-mlal-padal.c.in
index 272cb0a..12a9e46 100644
--- a/src/qs8-igemm/c16-neon-mlal-padal.c.in
+++ b/src/qs8-igemm/c16-neon-mlal-padal.c.in
@@ -115,12 +115,12 @@
int32x4_t vacc${M}x${ABC[N:N+4]} = vcombine_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]} );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -130,7 +130,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -158,11 +158,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in reversed(range(MR)):
$for N in range(0, NR, 16):
diff --git a/src/qs8-igemm/c2-neon-mull-padal-dup.c.in b/src/qs8-igemm/c2-neon-mull-padal-dup.c.in
index b142393..5d854f2 100644
--- a/src/qs8-igemm/c2-neon-mull-padal-dup.c.in
+++ b/src/qs8-igemm/c2-neon-mull-padal-dup.c.in
@@ -150,12 +150,12 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -165,7 +165,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -194,11 +194,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in reversed(range(MR)):
$for N in range(0, NR, 16):
diff --git a/src/qs8-igemm/c8-neon-mull-padal.c.in b/src/qs8-igemm/c8-neon-mull-padal.c.in
index 5ba099b..3f0aecc 100644
--- a/src/qs8-igemm/c8-neon-mull-padal.c.in
+++ b/src/qs8-igemm/c8-neon-mull-padal.c.in
@@ -133,12 +133,12 @@
int32x4_t vacc${M}x${ABC[N:N+4]} = vcombine_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]} );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -148,7 +148,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -176,11 +176,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in reversed(range(MR)):
$for N in range(0, NR, 16):
diff --git a/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 6926c15..937b755 100644
--- a/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -237,13 +237,13 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -255,7 +255,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -267,8 +267,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
index 8c3108f..822e036 100644
--- a/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -235,13 +235,13 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -253,7 +253,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -265,8 +265,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
index b9f356c..862974e 100644
--- a/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-igemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -231,13 +231,13 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -249,7 +249,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -261,8 +261,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
index 5d8934d..9137445 100644
--- a/src/qs8-igemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -190,13 +190,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -208,7 +208,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -219,8 +219,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index ff1e54a..cd8e764 100644
--- a/src/qs8-igemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -256,13 +256,13 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -274,7 +274,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -286,8 +286,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-igemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index bea4a9c..2e154a7 100644
--- a/src/qs8-igemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -168,13 +168,13 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -186,7 +186,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -198,8 +198,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x16c4-minmax-gemmlowp-neondot.c b/src/qs8-igemm/gen/1x16c4-minmax-gemmlowp-neondot.c
index ed5980e..33f23aa 100644
--- a/src/qs8-igemm/gen/1x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-igemm/gen/1x16c4-minmax-gemmlowp-neondot.c
@@ -106,13 +106,13 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -124,7 +124,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -136,8 +136,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
index cb1e65e..1e17928 100644
--- a/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
@@ -44,14 +44,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
index 902ca23..c785638 100644
--- a/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -263,13 +263,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -281,7 +281,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -292,8 +292,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
index 447f2f9..1495452 100644
--- a/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -174,13 +174,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -192,7 +192,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -203,8 +203,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/1x2-minmax-gemmlowp-scalar.c b/src/qs8-igemm/gen/1x2-minmax-gemmlowp-scalar.c
index 7e61341..64456f3 100644
--- a/src/qs8-igemm/gen/1x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-igemm/gen/1x2-minmax-gemmlowp-scalar.c
@@ -69,7 +69,7 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
@@ -77,24 +77,24 @@
const int32_t vq31product0x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x0 + vq31rounding) >> 31);
const int32_t vq31product0x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
diff --git a/src/qs8-igemm/gen/1x4-minmax-gemmlowp-scalar.c b/src/qs8-igemm/gen/1x4-minmax-gemmlowp-scalar.c
index f6e58e2..b8fd52a 100644
--- a/src/qs8-igemm/gen/1x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-igemm/gen/1x4-minmax-gemmlowp-scalar.c
@@ -75,7 +75,7 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -87,32 +87,32 @@
const int32_t vq31product0x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x2 + vq31rounding) >> 31);
const int32_t vq31product0x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
const int32_t vremainder0x3 = (vq31product0x3 & vremainder_mask) - (int32_t) (vq31product0x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
int32_t vout0x3 = asr_s32(vq31product0x3, vshift) + (int32_t) (vremainder0x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
vout0x3 = math_max_s32(vout0x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
vout0x3 = math_min_s32(vout0x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
index 6e59c58..661d8ba 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,23 +132,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
index ac9acf0..7d25a0b 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,23 +132,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
index 36debb2..c0dcabd 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -148,20 +148,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
index ebc1887..05f8476 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -148,20 +148,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
index 7e79ae7..4ea067c 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,23 +132,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
index 60cbccf..27eebfa 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,23 +132,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
index ee07d39..7cd1c20 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -148,20 +148,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
index 0b148d1..4ac0e38 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -148,20 +148,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
index f4a1138..93e6feb 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
@@ -123,8 +123,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,23 +137,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
index 3a55a85..86570d8 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
@@ -123,8 +123,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,23 +137,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
index 80623bc..9bad1ab 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
@@ -90,8 +90,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -104,23 +104,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
index fed83b2..5c58a64 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
@@ -92,8 +92,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -106,23 +106,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
index a9f0447..8429156 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -90,8 +90,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -120,20 +120,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
index c82935c..b7a1222 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -92,8 +92,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -122,20 +122,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
index 7620e4e..be97585 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -90,8 +90,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -104,23 +104,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
index 8c2619a..f5d3dcf 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -92,8 +92,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -106,23 +106,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
index 26c6287..8e085ec 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -90,8 +90,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -120,20 +120,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
index 1e1ea15..83d2a34 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -92,8 +92,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -122,20 +122,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index cf0aec9..c98411d 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -104,8 +104,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
@@ -113,22 +113,22 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc00x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc0x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc00x0123, vacc00x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index 71c1d3e..ec51265 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -100,8 +100,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
@@ -109,22 +109,22 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc00x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc0x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc00x0123, vacc00x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
index 8f9f58e..d9d0c90 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -109,23 +109,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
index 061e4fb..11011b1 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -111,23 +111,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index ffb8d1f..ded8e17 100644
--- a/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -166,11 +166,11 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -178,7 +178,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -188,8 +188,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
index 3eb5dc4..f6571b9 100644
--- a/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -165,11 +165,11 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -177,7 +177,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -187,8 +187,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
index 8f1b8ad..12af71f 100644
--- a/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-igemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -161,11 +161,11 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -173,7 +173,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -183,8 +183,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
index f15bf0b..3508bcf 100644
--- a/src/qs8-igemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -130,11 +130,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -142,7 +142,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -151,8 +151,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index a1735fe..81574a2 100644
--- a/src/qs8-igemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -172,11 +172,11 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -184,7 +184,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -194,8 +194,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-igemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index f90333e..d678e34 100644
--- a/src/qs8-igemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -124,11 +124,11 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -136,7 +136,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -146,8 +146,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/1x8c4-minmax-gemmlowp-neondot.c b/src/qs8-igemm/gen/1x8c4-minmax-gemmlowp-neondot.c
index 6178992..81bf586 100644
--- a/src/qs8-igemm/gen/1x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-igemm/gen/1x8c4-minmax-gemmlowp-neondot.c
@@ -92,11 +92,11 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -104,7 +104,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -114,8 +114,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qs8-igemm/gen/1x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..dc3a7a3
--- /dev/null
+++ b/src/qs8-igemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,159 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
+
+ vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-avx2.c b/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-avx2.c
index 262d505..34aedf9 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-avx2.c
@@ -104,8 +104,8 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -118,24 +118,24 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 141c892..3bdf444 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -171,11 +171,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -183,7 +183,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -192,8 +192,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
index 90b0238..34bf610 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -122,11 +122,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -134,7 +134,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -143,8 +143,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index c2ee2b1..c982ab3 100644
--- a/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -313,7 +313,7 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -323,7 +323,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -343,7 +343,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -361,8 +361,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
index 103a96b..6531145 100644
--- a/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -311,7 +311,7 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -321,7 +321,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -341,7 +341,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -359,8 +359,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
index 3b90389..ce22534 100644
--- a/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-igemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -335,7 +335,7 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -345,7 +345,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -365,7 +365,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -383,8 +383,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
index 25dede9..9fc9af7 100644
--- a/src/qs8-igemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -303,7 +303,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -313,7 +313,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -333,7 +333,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -350,8 +350,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 4a9edcb..07f01f8 100644
--- a/src/qs8-igemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -376,7 +376,7 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -386,7 +386,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -406,7 +406,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -424,8 +424,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-igemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index a742dc7..26f4388 100644
--- a/src/qs8-igemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -238,7 +238,7 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -248,7 +248,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -268,7 +268,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -286,8 +286,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
index 1469cbd..28f4d21 100644
--- a/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
@@ -48,14 +48,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
index 14c22de..677f8e1 100644
--- a/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -410,7 +410,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -420,7 +420,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -440,7 +440,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -457,8 +457,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
index eab9f86..6f18bb7 100644
--- a/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -271,7 +271,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -281,7 +281,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -301,7 +301,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -318,8 +318,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/2x2-minmax-gemmlowp-scalar.c b/src/qs8-igemm/gen/2x2-minmax-gemmlowp-scalar.c
index 08105fb..c8aa053 100644
--- a/src/qs8-igemm/gen/2x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-igemm/gen/2x2-minmax-gemmlowp-scalar.c
@@ -83,7 +83,7 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -95,32 +95,32 @@
const int32_t vq31product1x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x0 + vq31rounding) >> 31);
const int32_t vq31product1x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
const int32_t vremainder1x1 = (vq31product1x1 & vremainder_mask) - (int32_t) (vq31product1x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
int32_t vout1x1 = asr_s32(vq31product1x1, vshift) + (int32_t) (vremainder1x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
vout1x1 = math_max_s32(vout1x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
vout1x1 = math_min_s32(vout1x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-igemm/gen/2x4-minmax-gemmlowp-scalar.c b/src/qs8-igemm/gen/2x4-minmax-gemmlowp-scalar.c
index 983266a..05fb1ca 100644
--- a/src/qs8-igemm/gen/2x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-igemm/gen/2x4-minmax-gemmlowp-scalar.c
@@ -93,7 +93,7 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -113,7 +113,7 @@
const int32_t vq31product1x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x2 + vq31rounding) >> 31);
const int32_t vq31product1x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -123,8 +123,8 @@
const int32_t vremainder1x2 = (vq31product1x2 & vremainder_mask) - (int32_t) (vq31product1x2 < 0);
const int32_t vremainder1x3 = (vq31product1x3 & vremainder_mask) - (int32_t) (vq31product1x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -134,7 +134,7 @@
int32_t vout1x2 = asr_s32(vq31product1x2, vshift) + (int32_t) (vremainder1x2 > vremainder_threshold);
int32_t vout1x3 = asr_s32(vq31product1x3, vshift) + (int32_t) (vremainder1x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -144,7 +144,7 @@
vout1x2 = math_max_s32(vout1x2, vout_min);
vout1x3 = math_max_s32(vout1x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -154,7 +154,7 @@
vout1x2 = math_min_s32(vout1x2, vout_max);
vout1x3 = math_min_s32(vout1x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
index 3578a5b..33a075a 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,27 +167,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
index 5257f6e..2edacd9 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,27 +167,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
index 9e8349d..8dada86 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -191,24 +191,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
index c2f43d9..402b6d4 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -191,24 +191,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
index 0d80b2d..cce3b5f 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,27 +167,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
index 2108f4e..9d7733e 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,27 +167,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
index f3dd6b7..c91fcd9 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -191,24 +191,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
index ba9d551..f790aef 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -191,24 +191,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
index 428a117..51cc3cf 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
@@ -152,8 +152,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -172,27 +172,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
index 63d70b7..251dded 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
@@ -152,8 +152,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -172,27 +172,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
index dfebf12..ceed36c 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
@@ -112,8 +112,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,27 +132,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
index e5a1494..ca30e42 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
@@ -114,8 +114,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -134,27 +134,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
index f0dcbad..bc54486 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -112,8 +112,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -156,24 +156,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
index 1782540..c16c117 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -114,8 +114,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -158,24 +158,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
index 2832c06..cd2d9f2 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -112,8 +112,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,27 +132,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
index e48e958..6eb3515 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -114,8 +114,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -134,27 +134,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
index 95b32f2..727aac5 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -112,8 +112,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -156,24 +156,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
index fd6c95d..5bfabbd 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -114,8 +114,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -158,24 +158,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index 237b105..bc35e32 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -135,8 +135,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -148,24 +148,24 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc01x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index 9a67158..5b62a36 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -131,8 +131,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -144,24 +144,24 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc01x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
index 05804fa..d3a2605 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,27 +137,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
index 58a7df2..717b1ae 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -139,27 +139,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index be6906b..5ff93c2 100644
--- a/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -210,13 +210,13 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -228,7 +228,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -240,8 +240,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
index 49ed995..6dd31d6 100644
--- a/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -209,13 +209,13 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -227,7 +227,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -239,8 +239,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
index 9f6507a..e9e320b 100644
--- a/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-igemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -218,13 +218,13 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -236,7 +236,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -248,8 +248,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
index a41e6c8..2a692b4 100644
--- a/src/qs8-igemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -191,13 +191,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -209,7 +209,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -220,8 +220,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index ef3a726..8205735 100644
--- a/src/qs8-igemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -238,13 +238,13 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -256,7 +256,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -268,8 +268,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-igemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 8a38186..e499fe0 100644
--- a/src/qs8-igemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -164,13 +164,13 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -182,7 +182,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -194,8 +194,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qs8-igemm/gen/2x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..ab6220d
--- /dev/null
+++ b/src/qs8-igemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c1, vout_hi);
+ _mm_storeu_si32(c0, vout_lo);
+
+ c1 += 4;
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c1 += 2;
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c b/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c
index fa1ac3a..a5c416d 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c
@@ -127,8 +127,8 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -147,28 +147,28 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 958284a..ec1d49a 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -250,13 +250,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -268,7 +268,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -279,8 +279,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
index 4a15b90..bad3768 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -175,13 +175,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -193,7 +193,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -204,8 +204,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 3d06ac3..0a149f9 100644
--- a/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -389,7 +389,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -403,7 +403,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -431,7 +431,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -455,8 +455,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
index d2eb117..c5fef29 100644
--- a/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -387,7 +387,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -401,7 +401,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -429,7 +429,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -453,8 +453,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
index e42fdbd..1fa0999 100644
--- a/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-igemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -439,7 +439,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -453,7 +453,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -481,7 +481,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -505,8 +505,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
index baf2a7c..d427f34 100644
--- a/src/qs8-igemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -416,7 +416,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -430,7 +430,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -458,7 +458,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -481,8 +481,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 4171f7c..d105e81 100644
--- a/src/qs8-igemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -496,7 +496,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -510,7 +510,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -538,7 +538,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -562,8 +562,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-igemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 9a5f7c1..f1fb618 100644
--- a/src/qs8-igemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -308,7 +308,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -322,7 +322,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -350,7 +350,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -374,8 +374,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
index c9b90bb..db80256 100644
--- a/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
@@ -52,14 +52,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
index b44bd79..0f99704 100644
--- a/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -557,7 +557,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -571,7 +571,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -599,7 +599,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -622,8 +622,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
index 4d9e5b4..d73f4b3 100644
--- a/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -368,7 +368,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -382,7 +382,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -410,7 +410,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -433,8 +433,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/3x2-minmax-gemmlowp-scalar.c b/src/qs8-igemm/gen/3x2-minmax-gemmlowp-scalar.c
index 8589d8a..79b5895 100644
--- a/src/qs8-igemm/gen/3x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-igemm/gen/3x2-minmax-gemmlowp-scalar.c
@@ -97,7 +97,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -113,7 +113,7 @@
const int32_t vq31product2x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x0 + vq31rounding) >> 31);
const int32_t vq31product2x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
@@ -121,8 +121,8 @@
const int32_t vremainder2x0 = (vq31product2x0 & vremainder_mask) - (int32_t) (vq31product2x0 < 0);
const int32_t vremainder2x1 = (vq31product2x1 & vremainder_mask) - (int32_t) (vq31product2x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
@@ -130,7 +130,7 @@
int32_t vout2x0 = asr_s32(vq31product2x0, vshift) + (int32_t) (vremainder2x0 > vremainder_threshold);
int32_t vout2x1 = asr_s32(vq31product2x1, vshift) + (int32_t) (vremainder2x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
@@ -138,7 +138,7 @@
vout2x0 = math_max_s32(vout2x0, vout_min);
vout2x1 = math_max_s32(vout2x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
@@ -146,7 +146,7 @@
vout2x0 = math_min_s32(vout2x0, vout_max);
vout2x1 = math_min_s32(vout2x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-igemm/gen/3x4-minmax-gemmlowp-scalar.c b/src/qs8-igemm/gen/3x4-minmax-gemmlowp-scalar.c
index 1e0723f..b4b1ddc 100644
--- a/src/qs8-igemm/gen/3x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-igemm/gen/3x4-minmax-gemmlowp-scalar.c
@@ -111,7 +111,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -139,7 +139,7 @@
const int32_t vq31product2x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x2 + vq31rounding) >> 31);
const int32_t vq31product2x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -153,8 +153,8 @@
const int32_t vremainder2x2 = (vq31product2x2 & vremainder_mask) - (int32_t) (vq31product2x2 < 0);
const int32_t vremainder2x3 = (vq31product2x3 & vremainder_mask) - (int32_t) (vq31product2x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -168,7 +168,7 @@
int32_t vout2x2 = asr_s32(vq31product2x2, vshift) + (int32_t) (vremainder2x2 > vremainder_threshold);
int32_t vout2x3 = asr_s32(vq31product2x3, vshift) + (int32_t) (vremainder2x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -182,7 +182,7 @@
vout2x2 = math_max_s32(vout2x2, vout_min);
vout2x3 = math_max_s32(vout2x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -196,7 +196,7 @@
vout2x2 = math_min_s32(vout2x2, vout_max);
vout2x3 = math_min_s32(vout2x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
index 7c0ac28..0c941bd 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,15 +219,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
index ee31cdf..e98b019 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,15 +219,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
index ce17450..51b1271 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -234,7 +234,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -242,8 +242,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -251,12 +251,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
index 0bb45da..25b9a16 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -234,7 +234,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -242,8 +242,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -251,12 +251,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
index 9d3d199..02850f4 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,15 +219,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
index 71ad095..b886943 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,15 +219,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
index 8422a5d..c9743bc 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -234,7 +234,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -242,8 +242,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -251,12 +251,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
index 5e2ed5a..5b6f8e6 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -234,7 +234,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -242,8 +242,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -251,12 +251,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
index fd55767..7df63c6 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
@@ -181,8 +181,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -207,7 +207,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -215,8 +215,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -224,15 +224,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
index 39e2d25..197a6af 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
@@ -181,8 +181,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -207,7 +207,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -215,8 +215,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -224,15 +224,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
index 223cc60..b49c379 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
@@ -134,8 +134,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -160,7 +160,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -168,8 +168,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -177,15 +177,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
index a72c325..a7d9926 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
@@ -136,8 +136,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -162,7 +162,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -170,8 +170,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -179,15 +179,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
index 837a82a..a59b40a 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -134,8 +134,8 @@
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
__m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -192,7 +192,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -200,8 +200,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -209,12 +209,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
index 6d67db7..ca33a86 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -136,8 +136,8 @@
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
__m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -194,7 +194,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -202,8 +202,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -211,12 +211,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
index c0ca2dc..ca1f14f 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -134,8 +134,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -160,7 +160,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -168,8 +168,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -177,15 +177,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
index 2a8923f..d658603 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -136,8 +136,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -162,7 +162,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -170,8 +170,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -179,15 +179,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
index 722aaf7..1ec5500 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -134,8 +134,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -192,7 +192,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -200,8 +200,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -209,12 +209,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
index 9cdc776..ebf492b 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -136,8 +136,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -194,7 +194,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -202,8 +202,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -211,12 +211,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index 21a741d..1b89f99 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -166,8 +166,8 @@
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
const v128_t vacc2x01 = wasm_v32x4_shuffle(vacc2x0123, vsign2x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -183,27 +183,27 @@
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
const v128_t vq31prod2x0123 = wasm_v32x4_shuffle(vprod2x01, vprod2x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
const v128_t vrem2x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod2x0123, vremainder_mask), wasm_i32x4_lt(vq31prod2x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
vacc2x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod2x0123, vshift), wasm_i32x4_gt(vrem2x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vacc22x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc2x0123, vacc2x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc22x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index b5bddcb..cb3de49 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -162,8 +162,8 @@
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
const v128_t vacc2x01 = wasm_v32x4_shuffle(vacc2x0123, vsign2x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -179,27 +179,27 @@
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
const v128_t vq31prod2x0123 = wasm_v32x4_shuffle(vprod2x01, vprod2x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
const v128_t vrem2x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod2x0123, vremainder_mask), wasm_i32x4_lt(vq31prod2x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
vacc2x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod2x0123, vshift), wasm_i32x4_gt(vrem2x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vacc22x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc2x0123, vacc2x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc22x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
index eb89693..510b01b 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
@@ -139,8 +139,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -165,7 +165,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -173,8 +173,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -182,15 +182,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
index a42b121..3c38797 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
@@ -141,8 +141,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,7 +167,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -175,8 +175,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -184,15 +184,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
diff --git a/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 907b8a7..4464592 100644
--- a/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -254,7 +254,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -262,7 +262,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -278,7 +278,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -294,8 +294,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
index 8e61ea4..0348cbe 100644
--- a/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -253,7 +253,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -261,7 +261,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -277,7 +277,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -293,8 +293,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
index f6a2d9f..99036ad 100644
--- a/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-igemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -275,7 +275,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -283,7 +283,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -299,7 +299,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -315,8 +315,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 76cc0ca..7158ace 100644
--- a/src/qs8-igemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -252,7 +252,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -260,7 +260,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -276,7 +276,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -291,8 +291,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 86b9b9e..694efcd 100644
--- a/src/qs8-igemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -304,7 +304,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -312,7 +312,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -328,7 +328,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -344,8 +344,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-igemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 8937afe..916ab15 100644
--- a/src/qs8-igemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -204,7 +204,7 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -212,7 +212,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -228,7 +228,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -244,8 +244,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qs8-igemm/gen/3x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..ecd6239
--- /dev/null
+++ b/src/qs8-igemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,227 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ __m256i vacc2x01 = vacc0x01;
+ __m256i vacc2x23 = vacc0x23;
+ __m256i vacc2x45 = vacc0x45;
+ __m256i vacc2x67 = vacc0x67;
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a2));
+ const __m256i vxa2 = _mm256_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+ vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+ const __m256i vacc2x0213 = _mm256_hadd_epi32(vacc2x01, vacc2x23);
+ const __m256i vacc2x4657 = _mm256_hadd_epi32(vacc2x45, vacc2x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+ const __m256i vacc2x02461357 = _mm256_hadd_epi32(vacc2x0213, vacc2x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+ __m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+ __m256 vscaled2x01234567 = _mm256_cvtepi32_ps(vacc2x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+ vscaled2x01234567 = _mm256_mul_ps(vscaled2x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+ vacc2x01234567 = _mm256_cvtps_epi32(vscaled2x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+ __m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+ vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storeh_pi((__m64*) c2, _mm_castsi128_ps(vout_lo));
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout_lo, 2);
+ _mm_storeu_si32(c1, vout_hi);
+ _mm_storeu_si32(c0, vout_lo);
+
+ c2 += 4;
+ c1 += 4;
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout_lo, 4);
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c2 += 2;
+ c1 += 2;
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c b/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c
index 1bcb4ff..32d5ac2 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c
@@ -150,8 +150,8 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -176,7 +176,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -184,8 +184,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -193,7 +193,7 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
@@ -202,8 +202,8 @@
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 811d1c3..36540e9 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -329,7 +329,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -337,7 +337,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -353,7 +353,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -368,8 +368,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
index 85e9f9f..10bbd23 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -228,7 +228,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -236,7 +236,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -252,7 +252,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -267,8 +267,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 9edcdf6..b4b0e69 100644
--- a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -465,7 +465,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -483,7 +483,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -519,7 +519,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -549,8 +549,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
index 6bca2cd..d0be21a 100644
--- a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -463,7 +463,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -481,7 +481,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -517,7 +517,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -547,8 +547,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
index 364a7df..bbeb566 100644
--- a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -543,7 +543,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -561,7 +561,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -597,7 +597,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -627,8 +627,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
index af03024..141c850 100644
--- a/src/qs8-igemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -529,7 +529,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -547,7 +547,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -583,7 +583,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -612,8 +612,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 06e46b5..d6f1631 100644
--- a/src/qs8-igemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -616,7 +616,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -634,7 +634,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -670,7 +670,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -700,8 +700,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-igemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 0d1d136..deed830 100644
--- a/src/qs8-igemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -378,7 +378,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -396,7 +396,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -432,7 +432,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -462,8 +462,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16c4-minmax-gemmlowp-neondot.c b/src/qs8-igemm/gen/4x16c4-minmax-gemmlowp-neondot.c
index ac21adb..984d364 100644
--- a/src/qs8-igemm/gen/4x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-igemm/gen/4x16c4-minmax-gemmlowp-neondot.c
@@ -184,7 +184,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -202,7 +202,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -238,7 +238,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -268,8 +268,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
index 2d928f5..9974e69 100644
--- a/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
@@ -56,14 +56,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
index af30a3f..2f88c06 100644
--- a/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -704,7 +704,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -722,7 +722,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -758,7 +758,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -787,8 +787,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
index 35532ef..e255872 100644
--- a/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -465,7 +465,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -483,7 +483,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -519,7 +519,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -548,8 +548,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/4x2-minmax-gemmlowp-scalar.c b/src/qs8-igemm/gen/4x2-minmax-gemmlowp-scalar.c
index e6a7616..ffa8fbd 100644
--- a/src/qs8-igemm/gen/4x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-igemm/gen/4x2-minmax-gemmlowp-scalar.c
@@ -111,7 +111,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -131,7 +131,7 @@
const int32_t vq31product3x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x0 + vq31rounding) >> 31);
const int32_t vq31product3x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
@@ -141,8 +141,8 @@
const int32_t vremainder3x0 = (vq31product3x0 & vremainder_mask) - (int32_t) (vq31product3x0 < 0);
const int32_t vremainder3x1 = (vq31product3x1 & vremainder_mask) - (int32_t) (vq31product3x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
@@ -152,7 +152,7 @@
int32_t vout3x0 = asr_s32(vq31product3x0, vshift) + (int32_t) (vremainder3x0 > vremainder_threshold);
int32_t vout3x1 = asr_s32(vq31product3x1, vshift) + (int32_t) (vremainder3x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
@@ -162,7 +162,7 @@
vout3x0 = math_max_s32(vout3x0, vout_min);
vout3x1 = math_max_s32(vout3x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
@@ -172,7 +172,7 @@
vout3x0 = math_min_s32(vout3x0, vout_max);
vout3x1 = math_min_s32(vout3x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-igemm/gen/4x4-minmax-gemmlowp-scalar.c b/src/qs8-igemm/gen/4x4-minmax-gemmlowp-scalar.c
index 3d6256d..82f3bcc 100644
--- a/src/qs8-igemm/gen/4x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-igemm/gen/4x4-minmax-gemmlowp-scalar.c
@@ -129,7 +129,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -165,7 +165,7 @@
const int32_t vq31product3x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x2 + vq31rounding) >> 31);
const int32_t vq31product3x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -183,8 +183,8 @@
const int32_t vremainder3x2 = (vq31product3x2 & vremainder_mask) - (int32_t) (vq31product3x2 < 0);
const int32_t vremainder3x3 = (vq31product3x3 & vremainder_mask) - (int32_t) (vq31product3x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -202,7 +202,7 @@
int32_t vout3x2 = asr_s32(vq31product3x2, vshift) + (int32_t) (vremainder3x2 > vremainder_threshold);
int32_t vout3x3 = asr_s32(vq31product3x3, vshift) + (int32_t) (vremainder3x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -220,7 +220,7 @@
vout3x2 = math_max_s32(vout3x2, vout_min);
vout3x3 = math_max_s32(vout3x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -238,7 +238,7 @@
vout3x2 = math_min_s32(vout3x2, vout_max);
vout3x3 = math_min_s32(vout3x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
index a1df771..de5da17 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,15 +258,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
index 7011ea7..d4f6be1 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,15 +258,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
index 93acf70..3c7af53 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -277,7 +277,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -287,8 +287,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -298,12 +298,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
index 03607d8..e40ce90 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -277,7 +277,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -287,8 +287,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -298,12 +298,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
index ab00486..e577a64 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,15 +258,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
index 5530e47..2ab9bc8 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,15 +258,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
index 3c7ba08..f844b67 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -277,7 +277,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -287,8 +287,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -298,12 +298,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
index f261a0c..d8765d9 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -277,7 +277,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -287,8 +287,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -298,12 +298,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
index 3eb4e00..61b2117 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
@@ -210,8 +210,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -242,7 +242,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -252,8 +252,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -263,15 +263,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
index c610c59..fc69a9c 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
@@ -210,8 +210,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -242,7 +242,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -252,8 +252,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -263,15 +263,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
diff --git a/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index d7cd248..e0b5d5c 100644
--- a/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -298,7 +298,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -308,7 +308,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -328,7 +328,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -346,8 +346,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
index 3d94c7b..89b6e23 100644
--- a/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -297,7 +297,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -307,7 +307,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -327,7 +327,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -345,8 +345,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
index 8074e57..6af314f 100644
--- a/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-igemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -332,7 +332,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -342,7 +342,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -362,7 +362,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -380,8 +380,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 186e0a9..afef31c 100644
--- a/src/qs8-igemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -313,7 +313,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -323,7 +323,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -343,7 +343,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -360,8 +360,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 5e717ae..7c12248 100644
--- a/src/qs8-igemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -370,7 +370,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -380,7 +380,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -400,7 +400,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -418,8 +418,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-igemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 513cee6..405667f 100644
--- a/src/qs8-igemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -244,7 +244,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -254,7 +254,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -274,7 +274,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -292,8 +292,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x8c4-minmax-gemmlowp-neondot.c b/src/qs8-igemm/gen/4x8c4-minmax-gemmlowp-neondot.c
index 1f75974..3879c41 100644
--- a/src/qs8-igemm/gen/4x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-igemm/gen/4x8c4-minmax-gemmlowp-neondot.c
@@ -146,7 +146,7 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -156,7 +156,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -176,7 +176,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -194,8 +194,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-igemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 1495f51..fb38be2 100644
--- a/src/qs8-igemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -408,7 +408,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -418,7 +418,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -438,7 +438,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -455,8 +455,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-igemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
index 97817c4..efd7ec4 100644
--- a/src/qs8-igemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -281,7 +281,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -291,7 +291,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -311,7 +311,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -328,8 +328,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 91dd3df..a766833 100644
--- a/src/qs8-igemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -617,7 +617,7 @@
p -= 6 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -643,7 +643,7 @@
vacc5x89AB = vqrdmulhq_s32(vacc5x89AB, vmultiplier);
vacc5xCDEF = vqrdmulhq_s32(vacc5xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -695,7 +695,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -737,8 +737,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout5x0123456789ABCDEF = vmaxq_s8(vout5x0123456789ABCDEF, voutput_min);
vout4x0123456789ABCDEF = vmaxq_s8(vout4x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
index c645ef6..e03346b 100644
--- a/src/qs8-igemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -615,7 +615,7 @@
p -= 6 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -641,7 +641,7 @@
vacc5x89AB = vqrdmulhq_s32(vacc5x89AB, vmultiplier);
vacc5xCDEF = vqrdmulhq_s32(vacc5xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -693,7 +693,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -735,8 +735,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout5x0123456789ABCDEF = vmaxq_s8(vout5x0123456789ABCDEF, voutput_min);
vout4x0123456789ABCDEF = vmaxq_s8(vout4x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/6x16c4-minmax-gemmlowp-neondot.c b/src/qs8-igemm/gen/6x16c4-minmax-gemmlowp-neondot.c
index 7c9b9da..550da3c 100644
--- a/src/qs8-igemm/gen/6x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-igemm/gen/6x16c4-minmax-gemmlowp-neondot.c
@@ -236,7 +236,7 @@
p -= 6 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -262,7 +262,7 @@
vacc5x89AB = vqrdmulhq_s32(vacc5x89AB, vmultiplier);
vacc5xCDEF = vqrdmulhq_s32(vacc5xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -314,7 +314,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -356,8 +356,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout5x0123456789ABCDEF = vmaxq_s8(vout5x0123456789ABCDEF, voutput_min);
vout4x0123456789ABCDEF = vmaxq_s8(vout4x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-igemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 26da7be..be68e94 100644
--- a/src/qs8-igemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-igemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -386,7 +386,7 @@
p -= 6 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -400,7 +400,7 @@
vacc5x0123 = vqrdmulhq_s32(vacc5x0123, vmultiplier);
vacc5x4567 = vqrdmulhq_s32(vacc5x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -428,7 +428,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -452,8 +452,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout4x01234567_5x01234567 = vmaxq_s8(vout4x01234567_5x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-igemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
index df3e24c..565e3c1 100644
--- a/src/qs8-igemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -385,7 +385,7 @@
p -= 6 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -399,7 +399,7 @@
vacc5x0123 = vqrdmulhq_s32(vacc5x0123, vmultiplier);
vacc5x4567 = vqrdmulhq_s32(vacc5x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -427,7 +427,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -451,8 +451,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout4x01234567_5x01234567 = vmaxq_s8(vout4x01234567_5x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/6x8c4-minmax-gemmlowp-neondot.c b/src/qs8-igemm/gen/6x8c4-minmax-gemmlowp-neondot.c
index 1e4aab9..f778923 100644
--- a/src/qs8-igemm/gen/6x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-igemm/gen/6x8c4-minmax-gemmlowp-neondot.c
@@ -182,7 +182,7 @@
p -= 6 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -196,7 +196,7 @@
vacc5x0123 = vqrdmulhq_s32(vacc5x0123, vmultiplier);
vacc5x4567 = vqrdmulhq_s32(vacc5x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -224,7 +224,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -248,8 +248,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout4x01234567_5x01234567 = vmaxq_s8(vout4x01234567_5x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-igemm/gen/8x16c4-minmax-gemmlowp-neondot.c b/src/qs8-igemm/gen/8x16c4-minmax-gemmlowp-neondot.c
index e5c1331..3970c26 100644
--- a/src/qs8-igemm/gen/8x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-igemm/gen/8x16c4-minmax-gemmlowp-neondot.c
@@ -288,7 +288,7 @@
p -= 8 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -322,7 +322,7 @@
vacc7x89AB = vqrdmulhq_s32(vacc7x89AB, vmultiplier);
vacc7xCDEF = vqrdmulhq_s32(vacc7xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -390,7 +390,7 @@
vacc7x89AB = vrshlq_s32(vacc7x89AB, vright_shift);
vacc7xCDEF = vrshlq_s32(vacc7xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -444,8 +444,8 @@
int8x16_t vout6x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc6x89ABCDEF));
int8x16_t vout7x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc7x01234567), vqmovn_s16(vacc7x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout7x0123456789ABCDEF = vmaxq_s8(vout7x0123456789ABCDEF, voutput_min);
vout6x0123456789ABCDEF = vmaxq_s8(vout6x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-igemm/gen/8x8c4-minmax-gemmlowp-neondot.c b/src/qs8-igemm/gen/8x8c4-minmax-gemmlowp-neondot.c
index bc697f1..0b35cfc 100644
--- a/src/qs8-igemm/gen/8x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-igemm/gen/8x8c4-minmax-gemmlowp-neondot.c
@@ -218,7 +218,7 @@
p -= 8 * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -236,7 +236,7 @@
vacc7x0123 = vqrdmulhq_s32(vacc7x0123, vmultiplier);
vacc7x4567 = vqrdmulhq_s32(vacc7x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -272,7 +272,7 @@
vacc7x0123 = vrshlq_s32(vacc7x0123, vright_shift);
vacc7x4567 = vrshlq_s32(vacc7x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -302,8 +302,8 @@
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
int8x16_t vout6x01234567_7x01234567 = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc7x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout6x01234567_7x01234567 = vmaxq_s8(vout6x01234567_7x01234567, voutput_min);
vout4x01234567_5x01234567 = vmaxq_s8(vout4x01234567_5x01234567, voutput_min);
diff --git a/src/qs8-igemm/neon-mlal-lane.c.in b/src/qs8-igemm/neon-mlal-lane.c.in
index 33a6b16..5527bc2 100644
--- a/src/qs8-igemm/neon-mlal-lane.c.in
+++ b/src/qs8-igemm/neon-mlal-lane.c.in
@@ -184,12 +184,12 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -199,7 +199,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -228,11 +228,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in reversed(range(MR)):
$for N in range(0, NR, 16):
diff --git a/src/qs8-igemm/neon-mull-addw-dup.c.in b/src/qs8-igemm/neon-mull-addw-dup.c.in
index 65a3d13..145de0b 100644
--- a/src/qs8-igemm/neon-mull-addw-dup.c.in
+++ b/src/qs8-igemm/neon-mull-addw-dup.c.in
@@ -169,12 +169,12 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -184,7 +184,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -213,11 +213,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in reversed(range(MR)):
$for N in range(0, NR, 16):
diff --git a/src/qs8-igemm/scalar.c.in b/src/qs8-igemm/scalar.c.in
index 349de59..f26d11a 100644
--- a/src/qs8-igemm/scalar.c.in
+++ b/src/qs8-igemm/scalar.c.in
@@ -85,7 +85,7 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
$for M in range(MR):
$for N in range(NR):
const int64_t vproduct${M}x${N} = (int64_t) vacc${M}x${N} * (int64_t) vmultiplier;
@@ -95,28 +95,28 @@
$for N in range(NR):
const int32_t vq31product${M}x${N} = (int32_t) (uint32_t) ((uint64_t) (vproduct${M}x${N} + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
$for M in range(MR):
$for N in range(NR):
const int32_t vremainder${M}x${N} = (vq31product${M}x${N} & vremainder_mask) - (int32_t) (vq31product${M}x${N} < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
$for M in range(MR):
$for N in range(NR):
int32_t vout${M}x${N} = asr_s32(vq31product${M}x${N}, vshift) + (int32_t) (vremainder${M}x${N} > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} = math_max_s32(vout${M}x${N}, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} = math_min_s32(vout${M}x${N}, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} += voutput_zero_point;
diff --git a/src/xnnpack/dwconv.h b/src/xnnpack/dwconv.h
index 0b9f641..51b55bf 100644
--- a/src/xnnpack/dwconv.h
+++ b/src/xnnpack/dwconv.h
@@ -314,6 +314,9 @@
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x9__avx2_mul16)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up32x9__avx2_mul16)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__avx2_mul16)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up32x9__avx2_mul16)
+
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x9__sse41_mul32)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x9__sse41_mul32)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up24x9__sse41_mul32)
@@ -331,6 +334,11 @@
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up24x9__avx2_mul32)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up32x9__avx2_mul32)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up8x9__avx2_mul32)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__avx2_mul32)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up24x9__avx2_mul32)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up32x9__avx2_mul32)
+
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x9__avx512skx_mul32)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up32x9__avx512skx_mul32)
@@ -366,6 +374,9 @@
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x25__avx2_mul16)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up32x25__avx2_mul16)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__avx2_mul16)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up32x25__avx2_mul16)
+
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up8x25__sse41_mul32)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x25__sse41_mul32)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up24x25__sse41_mul32)
@@ -383,6 +394,11 @@
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up24x25__avx2_mul32)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up32x25__avx2_mul32)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up8x25__avx2_mul32)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__avx2_mul32)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up24x25__avx2_mul32)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up32x25__avx2_mul32)
+
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up16x25__avx512skx_mul32)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up32x25__avx512skx_mul32)
diff --git a/src/xnnpack/gemm.h b/src/xnnpack/gemm.h
index 0f79913..a1be4c0 100644
--- a/src/xnnpack/gemm.h
+++ b/src/xnnpack/gemm.h
@@ -726,6 +726,10 @@
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x8c8__avx2)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x8c8__avx2)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x8c8__avx2)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x8c8__avx2)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x8c8__avx2)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c8__avx512skx)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x16c8__avx512skx)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x16c8__avx512skx)
diff --git a/src/xnnpack/igemm.h b/src/xnnpack/igemm.h
index 6071bb9..d032731 100644
--- a/src/xnnpack/igemm.h
+++ b/src/xnnpack/igemm.h
@@ -529,6 +529,10 @@
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c8__avx2)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x8c8__avx2)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x8c8__avx2)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x8c8__avx2)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x8c8__avx2)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x16c8__avx512skx)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x16c8__avx512skx)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x16c8__avx512skx)
diff --git a/src/xnnpack/params-init.h b/src/xnnpack/params-init.h
index a245112..a1410cd 100644
--- a/src/xnnpack/params-init.h
+++ b/src/xnnpack/params-init.h
@@ -147,7 +147,7 @@
}
#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
-static inline void xnn_init_qs8_conv_minmax_scalar_params(
+static inline void xnn_init_qs8_conv_minmax_gemmlowp_scalar_params(
union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
float scale,
int8_t output_zero_point,
@@ -170,17 +170,17 @@
const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
const uint32_t remainder_threshold = remainder_mask >> 1;
- params->scalar.multiplier = multiplier;
- params->scalar.remainder_mask = (int32_t) remainder_mask;
- params->scalar.remainder_threshold = (int32_t) remainder_threshold;
- params->scalar.shift = (uint32_t) shift;
- params->scalar.output_min_less_zero_point = (int32_t) output_min - (int32_t) output_zero_point;
- params->scalar.output_max_less_zero_point = (int32_t) output_max - (int32_t) output_zero_point;
- params->scalar.output_zero_point = (int32_t) output_zero_point;
+ params->gemmlowp_scalar.multiplier = multiplier;
+ params->gemmlowp_scalar.remainder_mask = (int32_t) remainder_mask;
+ params->gemmlowp_scalar.remainder_threshold = (int32_t) remainder_threshold;
+ params->gemmlowp_scalar.shift = (uint32_t) shift;
+ params->gemmlowp_scalar.output_min_less_zero_point = (int32_t) output_min - (int32_t) output_zero_point;
+ params->gemmlowp_scalar.output_max_less_zero_point = (int32_t) output_max - (int32_t) output_zero_point;
+ params->gemmlowp_scalar.output_zero_point = (int32_t) output_zero_point;
}
#if XNN_ARCH_X86 || XNN_ARCH_X86_64
-static inline void xnn_init_qs8_conv_minmax_sse2_params(
+static inline void xnn_init_qs8_conv_minmax_gemmlowp_sse2_params(
union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
float scale,
int8_t output_zero_point,
@@ -202,30 +202,30 @@
const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
const uint32_t remainder_threshold = remainder_mask >> 1;
- params->sse2.multiplier[0] = multiplier;
- params->sse2.multiplier[1] = multiplier;
- params->sse2.multiplier[2] = multiplier;
- params->sse2.multiplier[3] = multiplier;
- params->sse2.rounding[0] = UINT64_C(0x40000000);
- params->sse2.rounding[1] = UINT64_C(0x40000000);
- params->sse2.remainder_mask[0] = (int32_t) remainder_mask;
- params->sse2.remainder_mask[1] = (int32_t) remainder_mask;
- params->sse2.remainder_mask[2] = (int32_t) remainder_mask;
- params->sse2.remainder_mask[3] = (int32_t) remainder_mask;
- params->sse2.remainder_threshold[0] = (int32_t) remainder_threshold;
- params->sse2.remainder_threshold[1] = (int32_t) remainder_threshold;
- params->sse2.remainder_threshold[2] = (int32_t) remainder_threshold;
- params->sse2.remainder_threshold[3] = (int32_t) remainder_threshold;
- params->sse2.shift[0] = (uint64_t) (uint32_t) shift;
- params->sse2.shift[1] = (uint64_t) (uint32_t) shift;
+ params->gemmlowp_sse2.multiplier[0] = multiplier;
+ params->gemmlowp_sse2.multiplier[1] = multiplier;
+ params->gemmlowp_sse2.multiplier[2] = multiplier;
+ params->gemmlowp_sse2.multiplier[3] = multiplier;
+ params->gemmlowp_sse2.rounding[0] = UINT64_C(0x40000000);
+ params->gemmlowp_sse2.rounding[1] = UINT64_C(0x40000000);
+ params->gemmlowp_sse2.remainder_mask[0] = (int32_t) remainder_mask;
+ params->gemmlowp_sse2.remainder_mask[1] = (int32_t) remainder_mask;
+ params->gemmlowp_sse2.remainder_mask[2] = (int32_t) remainder_mask;
+ params->gemmlowp_sse2.remainder_mask[3] = (int32_t) remainder_mask;
+ params->gemmlowp_sse2.remainder_threshold[0] = (int32_t) remainder_threshold;
+ params->gemmlowp_sse2.remainder_threshold[1] = (int32_t) remainder_threshold;
+ params->gemmlowp_sse2.remainder_threshold[2] = (int32_t) remainder_threshold;
+ params->gemmlowp_sse2.remainder_threshold[3] = (int32_t) remainder_threshold;
+ params->gemmlowp_sse2.shift[0] = (uint64_t) (uint32_t) shift;
+ params->gemmlowp_sse2.shift[1] = (uint64_t) (uint32_t) shift;
for (uint32_t i = 0; i < 8; i++) {
- params->sse2.output_zero_point[i] = (int16_t) output_zero_point;
- params->sse2.output_min[i] = (int16_t) output_min;
- params->sse2.output_max[i] = (int16_t) output_max;
+ params->gemmlowp_sse2.output_zero_point[i] = (int16_t) output_zero_point;
+ params->gemmlowp_sse2.output_min[i] = (int16_t) output_min;
+ params->gemmlowp_sse2.output_max[i] = (int16_t) output_max;
}
}
-static inline void xnn_init_qs8_conv_minmax_sse4_params(
+static inline void xnn_init_qs8_conv_minmax_gemmlowp_sse4_params(
union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
float scale,
int8_t output_zero_point,
@@ -247,32 +247,32 @@
const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
const uint32_t remainder_threshold = remainder_mask >> 1;
- params->sse4.multiplier[0] = multiplier;
- params->sse4.multiplier[1] = multiplier;
- params->sse4.multiplier[2] = multiplier;
- params->sse4.multiplier[3] = multiplier;
- params->sse4.rounding[0] = UINT64_C(0x40000000);
- params->sse4.rounding[1] = UINT64_C(0x40000000);
- params->sse4.remainder_mask[0] = (int32_t) remainder_mask;
- params->sse4.remainder_mask[1] = (int32_t) remainder_mask;
- params->sse4.remainder_mask[2] = (int32_t) remainder_mask;
- params->sse4.remainder_mask[3] = (int32_t) remainder_mask;
- params->sse4.remainder_threshold[0] = (int32_t) remainder_threshold;
- params->sse4.remainder_threshold[1] = (int32_t) remainder_threshold;
- params->sse4.remainder_threshold[2] = (int32_t) remainder_threshold;
- params->sse4.remainder_threshold[3] = (int32_t) remainder_threshold;
- params->sse4.shift[0] = (uint64_t) (uint32_t) shift;
- params->sse4.shift[1] = (uint64_t) (uint32_t) shift;
+ params->gemmlowp_sse4.multiplier[0] = multiplier;
+ params->gemmlowp_sse4.multiplier[1] = multiplier;
+ params->gemmlowp_sse4.multiplier[2] = multiplier;
+ params->gemmlowp_sse4.multiplier[3] = multiplier;
+ params->gemmlowp_sse4.rounding[0] = UINT64_C(0x40000000);
+ params->gemmlowp_sse4.rounding[1] = UINT64_C(0x40000000);
+ params->gemmlowp_sse4.remainder_mask[0] = (int32_t) remainder_mask;
+ params->gemmlowp_sse4.remainder_mask[1] = (int32_t) remainder_mask;
+ params->gemmlowp_sse4.remainder_mask[2] = (int32_t) remainder_mask;
+ params->gemmlowp_sse4.remainder_mask[3] = (int32_t) remainder_mask;
+ params->gemmlowp_sse4.remainder_threshold[0] = (int32_t) remainder_threshold;
+ params->gemmlowp_sse4.remainder_threshold[1] = (int32_t) remainder_threshold;
+ params->gemmlowp_sse4.remainder_threshold[2] = (int32_t) remainder_threshold;
+ params->gemmlowp_sse4.remainder_threshold[3] = (int32_t) remainder_threshold;
+ params->gemmlowp_sse4.shift[0] = (uint64_t) (uint32_t) shift;
+ params->gemmlowp_sse4.shift[1] = (uint64_t) (uint32_t) shift;
for (uint32_t i = 0; i < 8; i++) {
- params->sse4.output_zero_point[i] = (int16_t) output_zero_point;
+ params->gemmlowp_sse4.output_zero_point[i] = (int16_t) output_zero_point;
}
for (uint32_t i = 0; i < 16; i++) {
- params->sse4.output_min[i] = output_min;
- params->sse4.output_max[i] = output_max;
+ params->gemmlowp_sse4.output_min[i] = output_min;
+ params->gemmlowp_sse4.output_max[i] = output_max;
}
}
-static inline void xnn_init_qs8_conv_minmax_avx2_params(
+static inline void xnn_init_qs8_conv_minmax_gemmlowp_avx2_params(
union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
float scale,
int8_t output_zero_point,
@@ -295,32 +295,51 @@
const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
const uint32_t remainder_threshold = remainder_mask >> 1;
for (uint32_t i = 0; i < 8; i++) {
- params->avx2.multiplier[i] = multiplier;
+ params->gemmlowp_avx2.multiplier[i] = multiplier;
}
- params->avx2.rounding[0] = UINT64_C(0x40000000);
- params->avx2.rounding[1] = UINT64_C(0x40000000);
- params->avx2.rounding[2] = UINT64_C(0x40000000);
- params->avx2.rounding[3] = UINT64_C(0x40000000);
+ params->gemmlowp_avx2.rounding[0] = UINT64_C(0x40000000);
+ params->gemmlowp_avx2.rounding[1] = UINT64_C(0x40000000);
+ params->gemmlowp_avx2.rounding[2] = UINT64_C(0x40000000);
+ params->gemmlowp_avx2.rounding[3] = UINT64_C(0x40000000);
for (uint32_t i = 0; i < 8; i++) {
- params->avx2.remainder_mask[i] = (int32_t) remainder_mask;
- params->avx2.remainder_threshold[i] = (int32_t) remainder_threshold;
+ params->gemmlowp_avx2.remainder_mask[i] = (int32_t) remainder_mask;
+ params->gemmlowp_avx2.remainder_threshold[i] = (int32_t) remainder_threshold;
}
- params->avx2.shift[0] = (uint64_t) (uint32_t) shift;
- params->avx2.shift[1] = (uint64_t) (uint32_t) shift;
- params->avx2.shift[2] = (uint64_t) (uint32_t) shift;
- params->avx2.shift[3] = (uint64_t) (uint32_t) shift;
+ params->gemmlowp_avx2.shift[0] = (uint64_t) (uint32_t) shift;
+ params->gemmlowp_avx2.shift[1] = (uint64_t) (uint32_t) shift;
+ params->gemmlowp_avx2.shift[2] = (uint64_t) (uint32_t) shift;
+ params->gemmlowp_avx2.shift[3] = (uint64_t) (uint32_t) shift;
for (uint32_t i = 0; i < 16; i++) {
- params->avx2.output_zero_point[i] = (int16_t) output_zero_point;
+ params->gemmlowp_avx2.output_zero_point[i] = (int16_t) output_zero_point;
}
for (uint32_t i = 0; i < 32; i++) {
- params->avx2.output_min[i] = output_min;
- params->avx2.output_max[i] = output_max;
+ params->gemmlowp_avx2.output_min[i] = output_min;
+ params->gemmlowp_avx2.output_max[i] = output_max;
+ }
+}
+
+static inline void xnn_init_qs8_conv_minmax_fp32_avx2_params(
+ union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ int8_t output_zero_point,
+ int8_t output_min,
+ int8_t output_max)
+{
+ for (uint32_t i = 0; i < 8; i++) {
+ params->fp32_avx2.scale[i] = scale;
+ }
+ for (uint32_t i = 0; i < 16; i++) {
+ params->fp32_avx2.output_zero_point[i] = (int16_t) output_zero_point;
+ }
+ for (uint32_t i = 0; i < 32; i++) {
+ params->fp32_avx2.output_min[i] = output_min;
+ params->fp32_avx2.output_max[i] = output_max;
}
}
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
-static inline void xnn_init_qs8_conv_minmax_neon_params(
+static inline void xnn_init_qs8_conv_minmax_gemmlowp_neon_params(
union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
float scale,
int8_t output_zero_point,
@@ -340,16 +359,16 @@
assert(shift >= 0);
assert(shift < 32);
- params->neon.multiplier = multiplier;
- params->neon.right_shift = -shift;
- params->neon.output_zero_point = (int16_t) output_zero_point;
- params->neon.output_min = output_min;
- params->neon.output_max = output_max;
+ params->gemmlowp_neon.multiplier = multiplier;
+ params->gemmlowp_neon.right_shift = -shift;
+ params->gemmlowp_neon.output_zero_point = (int16_t) output_zero_point;
+ params->gemmlowp_neon.output_min = output_min;
+ params->gemmlowp_neon.output_max = output_max;
}
#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
#if XNN_ARCH_WASMSIMD
-static inline void xnn_init_qs8_conv_minmax_wasmsimd_params(
+static inline void xnn_init_qs8_conv_minmax_gemmlowp_wasmsimd_params(
union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
float scale,
int8_t output_zero_point,
@@ -372,25 +391,25 @@
const int64_t twice_multiplier = INT64_C(2) * (int64_t) multiplier;
const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
const uint32_t remainder_threshold = remainder_mask >> 1;
- params->wasmsimd.multiplier[0] = twice_multiplier;
- params->wasmsimd.multiplier[1] = twice_multiplier;
- params->wasmsimd.rounding[0] = INT64_C(0x80000000);
- params->wasmsimd.rounding[1] = INT64_C(0x80000000);
- params->wasmsimd.remainder_mask[0] = (int32_t) remainder_mask;
- params->wasmsimd.remainder_mask[1] = (int32_t) remainder_mask;
- params->wasmsimd.remainder_mask[2] = (int32_t) remainder_mask;
- params->wasmsimd.remainder_mask[3] = (int32_t) remainder_mask;
- params->wasmsimd.remainder_threshold[0] = (int32_t) remainder_threshold;
- params->wasmsimd.remainder_threshold[1] = (int32_t) remainder_threshold;
- params->wasmsimd.remainder_threshold[2] = (int32_t) remainder_threshold;
- params->wasmsimd.remainder_threshold[3] = (int32_t) remainder_threshold;
- params->wasmsimd.shift = shift;
+ params->gemmlowp_wasmsimd.multiplier[0] = twice_multiplier;
+ params->gemmlowp_wasmsimd.multiplier[1] = twice_multiplier;
+ params->gemmlowp_wasmsimd.rounding[0] = INT64_C(0x80000000);
+ params->gemmlowp_wasmsimd.rounding[1] = INT64_C(0x80000000);
+ params->gemmlowp_wasmsimd.remainder_mask[0] = (int32_t) remainder_mask;
+ params->gemmlowp_wasmsimd.remainder_mask[1] = (int32_t) remainder_mask;
+ params->gemmlowp_wasmsimd.remainder_mask[2] = (int32_t) remainder_mask;
+ params->gemmlowp_wasmsimd.remainder_mask[3] = (int32_t) remainder_mask;
+ params->gemmlowp_wasmsimd.remainder_threshold[0] = (int32_t) remainder_threshold;
+ params->gemmlowp_wasmsimd.remainder_threshold[1] = (int32_t) remainder_threshold;
+ params->gemmlowp_wasmsimd.remainder_threshold[2] = (int32_t) remainder_threshold;
+ params->gemmlowp_wasmsimd.remainder_threshold[3] = (int32_t) remainder_threshold;
+ params->gemmlowp_wasmsimd.shift = shift;
for (uint32_t i = 0; i < 8; i++) {
- params->wasmsimd.output_zero_point[i] = (int16_t) output_zero_point;
+ params->gemmlowp_wasmsimd.output_zero_point[i] = (int16_t) output_zero_point;
}
for (uint32_t i = 0; i < 16; i++) {
- params->wasmsimd.output_min[i] = output_min;
- params->wasmsimd.output_max[i] = output_max;
+ params->gemmlowp_wasmsimd.output_min[i] = output_min;
+ params->gemmlowp_wasmsimd.output_max[i] = output_max;
}
}
#endif // XNN_ARCH_WASMSIMD
diff --git a/src/xnnpack/params.h b/src/xnnpack/params.h
index 54955a4..4601652 100644
--- a/src/xnnpack/params.h
+++ b/src/xnnpack/params.h
@@ -291,7 +291,7 @@
int32_t output_min_less_zero_point;
int32_t output_max_less_zero_point;
int32_t output_zero_point;
- } scalar;
+ } gemmlowp_scalar;
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
struct {
int32_t multiplier;
@@ -299,7 +299,7 @@
int16_t output_zero_point;
int8_t output_min;
int8_t output_max;
- } neon;
+ } gemmlowp_neon;
#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
#if XNN_ARCH_X86 || XNN_ARCH_X86_64
struct {
@@ -311,7 +311,7 @@
XNN_ALIGN(16) int16_t output_zero_point[8];
XNN_ALIGN(16) int16_t output_min[8];
XNN_ALIGN(16) int16_t output_max[8];
- } sse2;
+ } gemmlowp_sse2;
struct {
XNN_ALIGN(16) uint32_t multiplier[4];
XNN_ALIGN(16) uint64_t rounding[2];
@@ -321,7 +321,7 @@
XNN_ALIGN(16) int16_t output_zero_point[8];
XNN_ALIGN(16) int8_t output_min[16];
XNN_ALIGN(16) int8_t output_max[16];
- } sse4;
+ } gemmlowp_sse4;
struct {
XNN_ALIGN(32) uint32_t multiplier[8];
XNN_ALIGN(32) uint64_t rounding[4];
@@ -331,7 +331,25 @@
XNN_ALIGN(32) int16_t output_zero_point[16];
XNN_ALIGN(32) int8_t output_min[32];
XNN_ALIGN(32) int8_t output_max[32];
- } avx2;
+ } gemmlowp_avx2;
+ struct {
+ XNN_ALIGN(16) float scale[4];
+ XNN_ALIGN(16) int16_t output_zero_point[8];
+ XNN_ALIGN(16) int16_t output_min[8];
+ XNN_ALIGN(16) int16_t output_max[8];
+ } fp32_sse2;
+ struct {
+ XNN_ALIGN(16) float scale[4];
+ XNN_ALIGN(16) int16_t output_zero_point[8];
+ XNN_ALIGN(16) int8_t output_min[16];
+ XNN_ALIGN(16) int8_t output_max[16];
+ } fp32_sse4;
+ struct {
+ XNN_ALIGN(32) float scale[8];
+ XNN_ALIGN(32) int16_t output_zero_point[16];
+ XNN_ALIGN(32) int8_t output_min[32];
+ XNN_ALIGN(32) int8_t output_max[32];
+ } fp32_avx2;
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
#if XNN_ARCH_WASMSIMD
struct {
@@ -343,7 +361,7 @@
XNN_ALIGN(16) int16_t output_zero_point[8];
XNN_ALIGN(16) int8_t output_min[16];
XNN_ALIGN(16) int8_t output_max[16];
- } wasmsimd;
+ } gemmlowp_wasmsimd;
#endif // XNN_ARCH_WASMSIMD
};