Blame - src/init.c - platform/external/XNNPACK

blob: c172e506393a4fcc87475a79659729c9511f2e8e [file] [log] [blame]

XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1	// Copyright (c) Facebook, Inc. and its affiliates.
				2	// All rights reserved.
				3	//
				4	// Copyright 2019 Google LLC
				5	//
				6	// This source code is licensed under the BSD-style license found in the
				7	// LICENSE file in the root directory of this source tree.
				8
Marat Dukhan	0184901	2020-04-27 19:28:32 -0700	[diff] [blame]	9	#include <math.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	10	#include <stdbool.h>
				11	#include <stddef.h>
				12	#include <stdint.h>
Marat Dukhan	04f03be	2019-11-19 12:36:47 -0800	[diff] [blame]	13	#include <string.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	14
Marat Dukhan	57133c0	2020-04-13 00:54:59 -0700	[diff] [blame]	15	#ifdef _WIN32
				16	#include <windows.h>
				17	#else
				18	#include <pthread.h>
				19	#endif
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	20
Marat Dukhan	496389f	2021-04-07 15:47:12 -0700	[diff] [blame]	21	#ifdef _MSC_VER
				22	#include <intrin.h>
				23	#endif
				24
Marat Dukhan	d343c22	2019-10-07 09:22:14 -0700	[diff] [blame]	25	#ifndef __EMSCRIPTEN__
				26	#include <cpuinfo.h>
				27	#endif
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	28
				29	#include <xnnpack.h>
Marat Dukhan	496389f	2021-04-07 15:47:12 -0700	[diff] [blame]	30	#include <xnnpack/allocator.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	31	#include <xnnpack/argmaxpool.h>
				32	#include <xnnpack/avgpool.h>
Marat Dukhan	1dadbf7	2019-10-01 10:46:20 -0700	[diff] [blame]	33	#include <xnnpack/common.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	34	#include <xnnpack/conv.h>
				35	#include <xnnpack/dwconv.h>
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	36	#include <xnnpack/depthtospace.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	37	#include <xnnpack/gavgpool.h>
				38	#include <xnnpack/gemm.h>
Marat Dukhan	4662b19	2020-05-21 15:52:03 -0700	[diff] [blame]	39	#include <xnnpack/fill.h>
Marat Dukhan	660fd19	2020-03-10 04:55:30 -0700	[diff] [blame]	40	#include <xnnpack/ibilinear.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	41	#include <xnnpack/igemm.h>
				42	#include <xnnpack/log.h>
				43	#include <xnnpack/lut.h>
				44	#include <xnnpack/maxpool.h>
				45	#include <xnnpack/pad.h>
				46	#include <xnnpack/params.h>
Marat Dukhan	c5a7a39	2021-05-21 16:04:31 -0700	[diff] [blame]	47	#include <xnnpack/params-init.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	48	#include <xnnpack/pavgpool.h>
				49	#include <xnnpack/prelu.h>
Marat Dukhan	1edc454	2020-01-27 12:40:13 -0800	[diff] [blame]	50	#include <xnnpack/raddstoreexpminusmax.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	51	#include <xnnpack/rmax.h>
				52	#include <xnnpack/spmm.h>
				53	#include <xnnpack/unpool.h>
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	54	#include <xnnpack/vaddsub.h>
Marat Dukhan	1e782c4	2019-11-21 17:02:40 -0800	[diff] [blame]	55	#include <xnnpack/vbinary.h>
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	56	#include <xnnpack/vcvt.h>
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	57	#include <xnnpack/vmul.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	58	#include <xnnpack/vmulcaddc.h>
Marat Dukhan	1e782c4	2019-11-21 17:02:40 -0800	[diff] [blame]	59	#include <xnnpack/vunary.h>
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	60	#include <xnnpack/zip.h>
				61
				62	#ifndef XNN_ENABLE_ASSEMBLY
				63	#define XNN_ENABLE_ASSEMBLY 1
				64	#endif
				65
Marat Dukhan	57133c0	2020-04-13 00:54:59 -0700	[diff] [blame]	66	#ifdef _WIN32
				67	static INIT_ONCE init_guard = INIT_ONCE_STATIC_INIT;
				68	#else
				69	static pthread_once_t init_guard = PTHREAD_ONCE_INIT;
				70	#endif
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	71
Marat Dukhan	496389f	2021-04-07 15:47:12 -0700	[diff] [blame]	72	static const struct xnn_allocator* volatile init_allocator = NULL;
				73
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	74	struct xnn_parameters xnn_params = {
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	75	.init_flags = 0
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	76	};
				77
Marat Dukhan	0184901	2020-04-27 19:28:32 -0700	[diff] [blame]	78	static void init(void) {
Marat Dukhan	a199d49	2020-07-24 15:01:25 -0700	[diff] [blame]	79	#if XNN_ARCH_WASM \|\| XNN_ARCH_WASMSIMD
Marat Dukhan	0184901	2020-04-27 19:28:32 -0700	[diff] [blame]	80	// Unlike most other architectures, on x86/x86-64 when floating-point instructions
				81	// have no NaN arguments, but produce NaN output, the output NaN has sign bit set.
				82	// We use it to distinguish x86/x86-64 from other architectures, by doing subtraction
				83	// of two infinities (must produce NaN per IEEE 754 standard).
				84	static const volatile float inf = INFINITY;
				85	const bool is_wasm_x86 = signbit(inf - inf);
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	86	#endif
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	87	uint32_t init_flags = XNN_INIT_FLAG_XNNPACK;
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	88
Marat Dukhan	1dadbf7	2019-10-01 10:46:20 -0700	[diff] [blame]	89	#if XNN_ARCH_ARM
Frank Barchard	bcdb1c1	2020-05-11 14:13:20 -0700	[diff] [blame]	90	#if XNN_PLATFORM_MOBILE
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	91	if (!cpuinfo_has_arm_neon()) {
				92	xnn_log_error("XNNPACK initialization failed: NEON is not supported");
				93	return;
				94	}
				95	#else
				96	if (!cpuinfo_has_arm_vfpv2() && !cpuinfo_has_arm_vfpv3()) {
				97	xnn_log_error("XNNPACK initialization failed: VFP is not supported");
				98	return;
				99	}
				100	#endif
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	101
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	102	if (cpuinfo_has_arm_neon()) {
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	103	/************************** QC8 micro-kernels **************************/
				104	#ifndef XNN_NO_QC8_OPERATORS
				105	init_flags \|= XNN_INIT_FLAG_QC8;
				106
				107	if (!XNN_PLATFORM_IOS && cpuinfo_has_arm_neon_dot()) {
				108	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x8c4__neondot);
				109	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c4__neondot);
				110	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x8c4__neondot);
				111	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c4__neondot);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	112	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	113	xnn_params.qc8.gemm.mr = 4;
				114	xnn_params.qc8.gemm.nr = 8;
				115	xnn_params.qc8.gemm.log2_kr = 2;
Frank Barchard	0bc5801	2021-11-22 18:12:05 -0800	[diff] [blame]	116	} else if (cpuinfo_has_arm_v8()) {
				117	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x8c2s4__neonv8_mlal);
				118	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x8c2s4__neonv8_mlal);
				119	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c2s4__neonv8_mlal);
				120	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c2s4__neonv8_mlal);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	121	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Frank Barchard	0bc5801	2021-11-22 18:12:05 -0800	[diff] [blame]	122	xnn_params.qc8.gemm.mr = 2;
				123	xnn_params.qc8.gemm.nr = 8;
				124	xnn_params.qc8.gemm.log2_kr = 1;
				125	xnn_params.qc8.gemm.log2_sr = 2;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	126	} else {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	127	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x8c2s4__neon_mlal);
				128	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x8c2s4__neon_mlal);
				129	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c2s4__neon_mlal);
				130	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c2s4__neon_mlal);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	131	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neon_params;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	132	xnn_params.qc8.gemm.mr = 2;
				133	xnn_params.qc8.gemm.nr = 8;
				134	xnn_params.qc8.gemm.log2_kr = 1;
Frank Barchard	e7043ff	2021-11-10 14:50:08 -0800	[diff] [blame]	135	xnn_params.qc8.gemm.log2_sr = 2;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	136	}
				137
Frank Barchard	0bc5801	2021-11-22 18:12:05 -0800	[diff] [blame]	138	if (cpuinfo_has_arm_neon_v8()) {
				139	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8_ld64;
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	140	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Frank Barchard	0bc5801	2021-11-22 18:12:05 -0800	[diff] [blame]	141	xnn_params.qc8.dwconv[0].channel_tile = 16;
				142	xnn_params.qc8.dwconv[0].primary_tile = 9;
				143	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8_ld64;
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	144	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Frank Barchard	0bc5801	2021-11-22 18:12:05 -0800	[diff] [blame]	145	xnn_params.qc8.dwconv[1].channel_tile = 8;
				146	xnn_params.qc8.dwconv[1].primary_tile = 25;
				147	} else {
				148	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8_ld64;
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	149	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_neon_params;
Frank Barchard	0bc5801	2021-11-22 18:12:05 -0800	[diff] [blame]	150	xnn_params.qc8.dwconv[0].channel_tile = 16;
				151	xnn_params.qc8.dwconv[0].primary_tile = 9;
				152	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8_ld64;
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	153	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_neon_params;
Frank Barchard	0bc5801	2021-11-22 18:12:05 -0800	[diff] [blame]	154	xnn_params.qc8.dwconv[1].channel_tile = 8;
				155	xnn_params.qc8.dwconv[1].primary_tile = 25;
				156	}
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	157	#endif // XNN_NO_QC8_OPERATORS
				158
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	159	/************************** QS8 micro-kernels **************************/
				160	#ifndef XNN_NO_QS8_OPERATORS
				161	init_flags \|= XNN_INIT_FLAG_QS8;
				162
Marat Dukhan	ec56b7e	2021-04-13 16:46:21 -0700	[diff] [blame]	163	if (!XNN_PLATFORM_IOS && cpuinfo_has_arm_neon_dot()) {
Marat Dukhan	e903dff	2021-07-16 19:43:41 -0700	[diff] [blame]	164	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x8c4__neondot);
				165	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x8c4__neondot);
				166	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x8c4__neondot);
				167	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x8c4__neondot);
				168	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Frank Barchard	fb0ab0b	2021-03-03 07:54:34 -0800	[diff] [blame]	169	xnn_params.qs8.gemm.mr = 4;
				170	xnn_params.qs8.gemm.nr = 8;
				171	xnn_params.qs8.gemm.log2_kr = 2;
				172	} else {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	173	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_2x8c2s4__neon_mlal);
				174	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_2x8c2s4__neon_mlal);
				175	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x8c2s4__neon_mlal);
				176	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x8c2s4__neon_mlal);
Marat Dukhan	e903dff	2021-07-16 19:43:41 -0700	[diff] [blame]	177	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Frank Barchard	fb0ab0b	2021-03-03 07:54:34 -0800	[diff] [blame]	178	xnn_params.qs8.gemm.mr = 2;
				179	xnn_params.qs8.gemm.nr = 8;
				180	xnn_params.qs8.gemm.log2_kr = 1;
Frank Barchard	66ae257	2021-11-02 17:36:21 -0700	[diff] [blame]	181	xnn_params.qs8.gemm.log2_sr = 2;
Frank Barchard	fb0ab0b	2021-03-03 07:54:34 -0800	[diff] [blame]	182	}
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	183
Frank Barchard	0d06573	2021-08-31 00:01:40 -0700	[diff] [blame]	184	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8_ld64;
Marat Dukhan	be18f5c	2021-07-16 18:46:39 -0700	[diff] [blame]	185	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Frank Barchard	0d06573	2021-08-31 00:01:40 -0700	[diff] [blame]	186	xnn_params.qs8.dwconv[0].channel_tile = 16;
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	187	xnn_params.qs8.dwconv[0].primary_tile = 9;
Marat Dukhan	5f2939f	2021-07-23 13:38:32 -0700	[diff] [blame]	188	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8_ld64;
Marat Dukhan	be18f5c	2021-07-16 18:46:39 -0700	[diff] [blame]	189	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	190	xnn_params.qs8.dwconv[1].channel_tile = 8;
				191	xnn_params.qs8.dwconv[1].primary_tile = 25;
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	192
				193	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				194	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__neon_c8_acc2,
				195	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__neon_c8_acc2,
				196	.mr = 7,
				197	};
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	198
				199	xnn_params.qs8.vadd = (struct vbinary_parameters) {
Marat Dukhan	01debd9	2021-07-29 18:14:21 -0700	[diff] [blame]	200	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__neon_ld64_x16,
				201	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x16,
				202	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x16,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	203	.init.qs8_addsub = xnn_init_qs8_add_minmax_neon_params,
Marat Dukhan	01debd9	2021-07-29 18:14:21 -0700	[diff] [blame]	204	.element_tile = 16,
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	205	};
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	206	if (cpuinfo_has_arm_neon_v8()) {
				207	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				208	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__neonv8_ld64_x16,
				209	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__neonv8_ld64_x16,
				210	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__neonv8_ld64_x16,
				211	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_neonv8_params,
				212	.element_tile = 16,
				213	};
				214	} else {
				215	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				216	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__neon_ld64_x16,
				217	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__neon_ld64_x16,
				218	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__neon_ld64_x16,
				219	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_neon_params,
				220	.element_tile = 16,
				221	};
				222	}
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	223	#endif // XNN_NO_QS8_OPERATORS
				224
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	225	/************************* QU8 micro-kernels *************************/
				226	#ifndef XNN_NO_QU8_OPERATORS
				227	init_flags \|= XNN_INIT_FLAG_QU8;
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	228
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	229	if (cpuinfo_has_arm_neon_dot()) {
Frank Barchard	de9c64a	2021-08-17 18:32:50 -0700	[diff] [blame]	230	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_2x16c4__neondot);
				231	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_2x16c4__neondot);
				232	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16c4__neondot);
				233	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16c4__neondot);
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	234	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
Frank Barchard	e033126	2021-08-11 23:18:59 -0700	[diff] [blame]	235	xnn_params.qu8.gemm.mr = 2;
Frank Barchard	de9c64a	2021-08-17 18:32:50 -0700	[diff] [blame]	236	xnn_params.qu8.gemm.nr = 16;
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	237	xnn_params.qu8.gemm.log2_kr = 2;
				238	} else {
				239	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x8__neon_mlal_lane);
				240	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x8__neon_mlal_lane);
				241	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x8__neon_mlal_lane);
				242	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x8__neon_mlal_lane);
				243	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				244	xnn_params.qu8.gemm.mr = 4;
				245	xnn_params.qu8.gemm.nr = 8;
				246	}
Frank Barchard	354cbc6	2021-09-27 21:42:41 -0700	[diff] [blame]	247	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mul8;
Marat Dukhan	73a899a	2021-07-27 00:10:38 -0700	[diff] [blame]	248	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
Frank Barchard	0d06573	2021-08-31 00:01:40 -0700	[diff] [blame]	249	xnn_params.qu8.dwconv[0].channel_tile = 16;
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	250	xnn_params.qu8.dwconv[0].primary_tile = 9;
Frank Barchard	354cbc6	2021-09-27 21:42:41 -0700	[diff] [blame]	251	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mul8;
Marat Dukhan	73a899a	2021-07-27 00:10:38 -0700	[diff] [blame]	252	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
Marat Dukhan	43b46ee	2021-07-15 19:07:50 -0700	[diff] [blame]	253	xnn_params.qu8.dwconv[1].channel_tile = 8;
				254	xnn_params.qu8.dwconv[1].primary_tile = 25;
				255
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	256	xnn_params.qu8.avgpool = (struct avgpool_parameters) {
				257	.up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__neon_c8,
				258	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__neon_c8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	259	.mr = 9,
				260	.qr = 8,
				261	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	262	xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
				263	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__neon_c8,
				264	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__neon_c8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	265	.mr = 7,
				266	};
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	267	xnn_params.qu8.vadd = (struct vbinary_parameters) {
Frank Barchard	0a3093c	2021-08-31 09:58:11 -0700	[diff] [blame]	268	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__neon_ld64_x16,
				269	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__neon_ld64_x16,
				270	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__neon_ld64_x16,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	271	.init.qu8_addsub = xnn_init_qu8_add_minmax_neon_params,
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	272	.element_tile = 8,
				273	};
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	274	if (cpuinfo_has_arm_neon_v8()) {
				275	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				276	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__neonv8_ld64_x16,
				277	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__neonv8_ld64_x16,
				278	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__neonv8_ld64_x16,
				279	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_neonv8_params,
				280	.element_tile = 16,
				281	};
				282	} else {
				283	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				284	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__neon_ld64_x16,
				285	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__neon_ld64_x16,
				286	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__neon_ld64_x16,
				287	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_neon_params,
				288	.element_tile = 16,
				289	};
				290	}
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	291	#endif // XNN_NO_QU8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	292
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	293	/************************** S8 micro-kernels **************************/
				294	#ifndef XNN_NO_S8_OPERATORS
				295	init_flags \|= XNN_INIT_FLAG_S8;
				296
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	297	xnn_params.s8.clamp = (struct vunary_parameters) {
				298	.ukernel = (xnn_univector_ukernel_function) xnn_s8_vclamp_ukernel__neon_x64,
				299	.init.s8_minmax = xnn_init_s8_minmax_neon_params,
				300	.element_tile = 64,
				301	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	302	xnn_params.s8.ibilinear = (struct ibilinear_parameters) {
				303	.ukernel = (xnn_ibilinear_ukernel_function) xnn_s8_ibilinear_ukernel__neon_c8,
				304	.pixel_tile = 1,
				305	.channel_tile = 8,
				306	};
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	307	xnn_params.s8.maxpool = (struct maxpool_parameters) {
				308	.ukernel = (xnn_maxpool_ukernel_function) xnn_s8_maxpool_minmax_ukernel_9p8x__neon_c16,
Marat Dukhan	dc5c148	2021-08-16 09:03:15 -0700	[diff] [blame]	309	.init.s8 = xnn_init_s8_minmax_neon_params,
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	310	.mr = 9,
				311	.qr = 8,
				312	};
				313	#endif // XNN_NO_S8_OPERATORS
				314
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	315	/************************** U8 micro-kernels **************************/
				316	#ifndef XNN_NO_U8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	317	init_flags \|= XNN_INIT_FLAG_U8;
				318
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	319	xnn_params.u8.clamp = (struct vunary_parameters) {
				320	.ukernel = (xnn_univector_ukernel_function) xnn_u8_vclamp_ukernel__neon_x64,
				321	.init.u8_minmax = xnn_init_u8_minmax_neon_params,
				322	.element_tile = 64,
				323	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	324	xnn_params.u8.ibilinear = (struct ibilinear_parameters) {
				325	.ukernel = (xnn_ibilinear_ukernel_function) xnn_u8_ibilinear_ukernel__neon_c8,
				326	.pixel_tile = 1,
				327	.channel_tile = 8,
				328	};
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	329	xnn_params.u8.maxpool = (struct maxpool_parameters) {
				330	.ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__neon_c16,
Marat Dukhan	2ea50a0	2021-08-16 12:59:19 -0700	[diff] [blame]	331	.init.u8 = xnn_init_u8_minmax_neon_params,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	332	.mr = 9,
				333	.qr = 8,
				334	};
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	335	xnn_params.u8.rmax = xnn_u8_rmax_ukernel__neon;
				336	xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
				337	#endif // XNN_NO_U8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	338
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	339	/************************** X8 micro-kernels **************************/
				340	#ifndef XNN_NO_X8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	341	init_flags \|= XNN_INIT_FLAG_X8;
				342
Marat Dukhan	d67539d	2021-09-08 23:06:03 -0700	[diff] [blame]	343	xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar_x4;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	344	xnn_params.x8.zip = (struct zip_parameters) {
				345	.x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__neon,
				346	.x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__neon,
				347	.x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__neon,
				348	.xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__neon,
				349	};
				350	#endif // XNN_NO_X8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	351
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	352	/************************** F32 micro-kernels **************************/
				353	#ifndef XNN_NO_F32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	354	init_flags \|= XNN_INIT_FLAG_F32;
				355
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	356	#if XNN_ENABLE_ASSEMBLY
				357	switch (cpuinfo_get_uarch(0)->uarch) {
				358	case cpuinfo_uarch_cortex_a5:
				359	case cpuinfo_uarch_cortex_a7:
Frank Barchard	490febe	2020-07-16 18:42:17 -0700	[diff] [blame]	360	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a7);
				361	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a7);
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	362	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
				363	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	364	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	365	xnn_params.f32.gemm.mr = 4;
				366	xnn_params.f32.gemm.nr = 8;
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	367	break;
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	368
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	369	case cpuinfo_uarch_cortex_a53:
				370	case cpuinfo_uarch_cortex_a55r0:
				371	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53);
				372	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53);
				373	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
				374	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	375	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	376	xnn_params.f32.gemm.mr = 4;
				377	xnn_params.f32.gemm.nr = 8;
				378	break;
				379
Frank Barchard	f975ee0	2021-11-05 16:01:00 -0700	[diff] [blame]	380	case cpuinfo_uarch_cortex_a35:
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	381	case cpuinfo_uarch_cortex_a55:
				382	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55);
				383	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55);
				384	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
				385	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	386	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	387	xnn_params.f32.gemm.mr = 4;
				388	xnn_params.f32.gemm.nr = 8;
				389	break;
				390
				391	case cpuinfo_uarch_cortex_a57:
				392	case cpuinfo_uarch_cortex_a72:
				393	case cpuinfo_uarch_cortex_a73:
				394	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_pld_cortex_a75);
				395	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_pld_cortex_a75);
				396	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
				397	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	398	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	399	xnn_params.f32.gemm.mr = 4;
				400	xnn_params.f32.gemm.nr = 8;
				401	break;
				402
				403	case cpuinfo_uarch_krait:
				404	default:
				405	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a75);
				406	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a75);
				407	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
				408	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	409	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	410	xnn_params.f32.gemm.mr = 4;
				411	xnn_params.f32.gemm.nr = 8;
				412	break;
				413	}
				414	#if XNN_MAX_UARCH_TYPES > 1
				415	{
				416	/* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
				417	const uint32_t mr = xnn_params.f32.gemm.mr;
				418	const uint32_t nr = xnn_params.f32.gemm.nr;
				419	const uint32_t log2_sr = xnn_params.f32.gemm.log2_sr;
				420	for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
				421	const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
				422	if (uarch_info == NULL) {
				423	/* No more microarchitectures in the system */
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	424	break;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	425	}
				426
				427	switch (uarch_info->uarch) {
				428	case cpuinfo_uarch_cortex_a53:
				429	case cpuinfo_uarch_cortex_a55r0:
				430	if (mr == 4 && nr == 8 && log2_sr == 0) {
				431	xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53;
				432	xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53;
				433	xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64;
				434	xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64;
				435	}
				436	break;
				437	case cpuinfo_uarch_cortex_a55:
				438	if (mr == 4 && nr == 8 && log2_sr == 0) {
				439	xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55;
				440	xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55;
				441	xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64;
				442	xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64;
				443	}
				444	break;
				445	default:
				446	break;
				447	}
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	448	}
				449	}
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	450	#endif // XNN_MAX_UARCH_TYPES > 1
				451	#else // XNN_ENABLE_ASSEMBLY
				452	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__neon_lane_ld128);
				453	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__neon_lane_ld128);
				454	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
				455	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	456	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	457	xnn_params.f32.gemm.mr = 4;
				458	xnn_params.f32.gemm.nr = 8;
				459	#endif // XNN_ENABLE_ASSEMBLY
				460	xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__neon_lane_ld64);
				461	xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__neon_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	462	xnn_params.f32.gemm2.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	463	xnn_params.f32.gemm2.mr = 4;
				464	xnn_params.f32.gemm2.nr = 2;
				465
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	466	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x3__neon;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	467	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_scalar_params;
Frank Barchard	dbe781b	2021-10-18 10:29:52 -0700	[diff] [blame]	468	xnn_params.f32.dwconv[0].channel_tile = 8,
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	469	xnn_params.f32.dwconv[0].primary_tile = 3,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	470
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	471	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x4__neon;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	472	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_scalar_params;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	473	xnn_params.f32.dwconv[1].channel_tile = 8,
				474	xnn_params.f32.dwconv[1].primary_tile = 4,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	475
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	476	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__neon;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	477	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
Frank Barchard	dbe781b	2021-10-18 10:29:52 -0700	[diff] [blame]	478	xnn_params.f32.dwconv[2].channel_tile = 8;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	479	xnn_params.f32.dwconv[2].primary_tile = 9;
				480
				481	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x25__neon_acc2;
				482	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_scalar_params;
				483	xnn_params.f32.dwconv[3].channel_tile = 8;
				484	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	485
				486	xnn_params.f32.avgpool = (struct avgpool_parameters) {
				487	.up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__neon_c4,
				488	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__neon_c4,
				489	.mr = 9,
				490	.qr = 8,
				491	};
				492	xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
				493	.up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__neon_c4,
				494	.mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__neon_c4,
				495	.mr = 9,
				496	.qr = 8,
				497	};
				498	xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
				499	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__neon_c4,
				500	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__neon_c4,
				501	.mr = 7,
				502	};
				503	xnn_params.f32.maxpool = (struct maxpool_parameters) {
				504	.ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__neon_c4,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	505	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	506	.mr = 9,
				507	.qr = 8,
				508	};
				509	xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhan	ef25c6d	2020-07-24 00:59:40 -0700	[diff] [blame]	510	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__neon_c4,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	511	.mr = 4,
				512	};
				513	xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhan	ef25c6d	2020-07-24 00:59:40 -0700	[diff] [blame]	514	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__neon_c4,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	515	.mr = 9,
				516	};
				517	xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhan	ef25c6d	2020-07-24 00:59:40 -0700	[diff] [blame]	518	.mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__neon_c4,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	519	.mr = 9,
				520	.qr = 8,
				521	};
				522	xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
				523	.ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__neon_c8,
				524	.pixel_tile = 1,
				525	.channel_tile = 8,
				526	};
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	527	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__neon_x8;
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	528	xnn_params.f32.clamp = (struct vunary_parameters) {
				529	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__neon_x8,
				530	.init.f32_minmax = xnn_init_f32_minmax_scalar_params,
				531	.element_tile = 8,
				532	};
Marat Dukhan	b6bd4bc	2020-12-01 17:01:40 -0800	[diff] [blame]	533	if (cpuinfo_has_arm_neon_fma()) {
				534	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__neonfma_rr1_p6_x8;
				535	} else {
				536	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__neon_rr2_lut16_p3_x8;
				537	}
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	538	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__neon_x16;
Marat Dukhan	2881333	2020-06-10 18:05:38 -0700	[diff] [blame]	539	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__neon_x8;
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	540	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__neon_x8;
Marat Dukhan	64e5251	2020-06-09 13:41:16 -0700	[diff] [blame]	541	if (cpuinfo_has_arm_neon_v8()) {
				542	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__neonv8_x8;
				543	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__neonv8_x8;
				544	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__neonv8_x8;
				545	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__neonv8_x8;
				546	} else {
				547	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__neon_x8;
				548	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__neon_x8;
				549	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__neon_x8;
				550	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__neon_x8;
				551	}
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	552	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__neon_rr2_lut64_p2_nr2recps_x8;
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	553	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__neon_x8;
Marat Dukhan	6804bbd	2020-06-30 19:26:11 -0700	[diff] [blame]	554	xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__scalar_sqrt_x1;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	555	xnn_params.f32.prelu = (struct prelu_parameters) {
				556	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__neon_2x8,
				557	.row_tile = 2,
				558	.channel_tile = 8,
				559	};
				560	xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__neon_lut64_p2_x8;
				561	xnn_params.f32.rmax = xnn_f32_rmax_ukernel__neon;
				562	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	563	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__neon_x8,
				564	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__neon_x8,
				565	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__neon_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	566	.element_tile = 8,
				567	};
				568	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	569	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__scalar_x2,
				570	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__scalar_x2,
				571	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__scalar_x2,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	572	.element_tile = 2,
				573	};
				574	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	575	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__neon_x8,
				576	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__neon_x8,
				577	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__neon_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	578	.element_tile = 8,
				579	};
				580	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	581	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__neon_x8,
				582	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__neon_x8,
				583	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__neon_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	584	.element_tile = 8,
				585	};
				586	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	587	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__neon_x8,
				588	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__neon_x8,
				589	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__neon_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	590	.element_tile = 8,
				591	};
				592	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	593	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__neon_x8,
				594	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__neon_x8,
				595	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__neon_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	596	.element_tile = 8,
				597	};
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	598	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	599	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__neon_x8,
				600	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__neon_x8,
				601	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__neon_x8,
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	602	.element_tile = 8,
				603	};
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	604	xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan	9531e9f	2020-07-24 15:25:02 -0700	[diff] [blame]	605	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__neon_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	606	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	607	.channel_tile = 4,
				608	.row_tile = 2,
				609	};
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	610	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	cfbed0a	2020-12-08 10:01:51 -0800	[diff] [blame]	611	init_flags \|= XNN_INIT_FLAG_CHW_OPT;
				612
Marat Dukhan	3e91338	2020-12-07 13:36:08 -0800	[diff] [blame]	613	xnn_params.f32.spmm = (struct spmm_parameters) {
				614	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_32x1__neon,
				615	.mr = 32,
				616	.nr = 1,
				617	};
Marat Dukhan	c763488	2020-12-07 15:11:12 -0800	[diff] [blame]	618	xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
				619	.ukernel_with_symm_padding =
				620	(xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__neon_2x2,
				621	.output_channel_tile = 4,
				622	.output_height_tile = 2,
				623	.output_width_tile = 2,
				624	};
Marat Dukhan	3e91338	2020-12-07 13:36:08 -0800	[diff] [blame]	625	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
				626	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__neon_2x4,
Marat Dukhan	3e91338	2020-12-07 13:36:08 -0800	[diff] [blame]	627	.output_width_tile = 4,
				628	.output_height_tile = 2,
				629	};
				630	xnn_params.f32.dwconv2d_chw_3x3s2 = (struct dwconv2d_chw_parameters) {
				631	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__neon_1x4,
Marat Dukhan	3e91338	2020-12-07 13:36:08 -0800	[diff] [blame]	632	.output_width_tile = 4,
				633	.output_height_tile = 1,
				634	};
				635	xnn_params.f32.dwconv2d_chw_5x5 = (struct dwconv2d_chw_parameters) {
				636	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5p2__neon_1x4,
Marat Dukhan	3e91338	2020-12-07 13:36:08 -0800	[diff] [blame]	637	.output_width_tile = 4,
				638	.output_height_tile = 1,
				639	};
				640	xnn_params.f32.dwconv2d_chw_5x5s2 = (struct dwconv2d_chw_parameters) {
				641	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4,
Marat Dukhan	3e91338	2020-12-07 13:36:08 -0800	[diff] [blame]	642	.output_width_tile = 4,
				643	.output_height_tile = 1,
				644	};
				645	xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
				646	.ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__neon_x4,
				647	.channel_tile = 4,
				648	};
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	649	xnn_params.f32.ibilinear_chw = (struct ibilinear_chw_parameters) {
Artsiom Ablavatski	2202c81	2021-01-22 14:16:43 -0800	[diff] [blame]	650	.ukernel = (xnn_ibilinear_chw_ukernel_function) xnn_f32_ibilinear_chw_ukernel__neon_p8,
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	651	.channel_tile = 1,
Artsiom Ablavatski	b3ffd58	2021-03-31 13:00:08 -0700	[diff] [blame]	652	.pixel_tile = 8,
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	653	};
				654	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	655	#endif // XNN_NO_F32_OPERATORS
				656
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	657	/************************* VCVT micro-kernels *************************/
				658	#ifndef XNN_NO_VCVT_OPERATORS
				659	init_flags \|= XNN_INIT_FLAG_VCVT;
				660
				661	if (cpuinfo_has_arm_neon_fp16()) {
				662	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__neonfp16_x16;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	663	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__neonfp16_x16;
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	664	} else {
				665	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__neon_int16_x16;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	666	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__neon_x8;
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	667	}
Marat Dukhan	ed2d776	2021-12-03 23:51:19 -0800	[diff] [blame]	668	if (cpuinfo_has_arm_neon_v8()) {
				669	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
				670	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qs8_vcvt_ukernel__neonv8_x32,
				671	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_neonv8_params,
				672	.element_tile = 32,
				673	};
				674	xnn_params.vcvt.f32_to_qu8 = (struct vunary_parameters) {
				675	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__neonv8_x32,
				676	.init.f32_qu8_cvt = xnn_init_f32_qu8_cvt_neonv8_params,
				677	.element_tile = 32,
				678	};
				679	} else {
				680	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
				681	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qs8_vcvt_ukernel__neon_x32,
				682	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_neon_params,
				683	.element_tile = 32,
				684	};
				685	xnn_params.vcvt.f32_to_qu8 = (struct vunary_parameters) {
				686	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__neon_x32,
				687	.init.f32_qu8_cvt = xnn_init_f32_qu8_cvt_neon_params,
				688	.element_tile = 32,
				689	};
				690	}
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	691	#endif // XNN_NO_VCVT_OPERATORS
				692
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	693	/************************** X32 micro-kernels **************************/
				694	#ifndef XNN_NO_X32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	695	init_flags \|= XNN_INIT_FLAG_X32;
				696
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	697	xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__neon;
				698	xnn_params.x32.zip = (struct zip_parameters) {
				699	.x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__neon,
				700	.x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__neon,
				701	.x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__neon,
				702	.xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__neon,
				703	};
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	704	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	ad71b9a	2020-11-20 00:01:51 -0800	[diff] [blame]	705	xnn_params.x32.depthtospace2d_chw2hwc = (struct depthtospace2d_chw2hwc_parameters) {
				706	.ukernel = (xnn_depthtospace2d_chw2hwc_ukernel_function) xnn_x32_depthtospace2d_chw2hwc_ukernel__scalar,
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	707	.channel_tile = 1,
				708	.pixel_tile = 1,
				709	};
				710	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	711	#endif // XNN_NO_X32_OPERATORS
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	712
				713	/************************** XX micro-kernels **************************/
				714	#ifndef XNN_NO_XX_OPERATORS
				715	init_flags \|= XNN_INIT_FLAG_XX;
				716
				717	xnn_params.xx.copy = (xnn_univector_ukernel_function) xnn_xx_copy_ukernel__memcpy;
				718	xnn_params.xx.fill = (struct fill_parameters) {
				719	.ukernel = (xnn_fill_ukernel_function) xnn_xx_fill_ukernel__neon_x64,
				720	.row_tile = 1,
				721	};
Marat Dukhan	0461f2d	2021-08-08 12:36:29 -0700	[diff] [blame]	722	xnn_params.xx.pad = (struct pad_parameters) {
				723	.ukernel = (xnn_pad_ukernel_function) xnn_xx_pad_ukernel__neon,
				724	.row_tile = 1,
				725	};
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	726	#endif // XNN_NO_XX_OPERATORS
				727
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	728	} else if (!XNN_PLATFORM_MOBILE) {
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	729
Marat Dukhan	3b9b4bc	2021-08-24 00:23:07 -0700	[diff] [blame]	730	/************************* QS8 micro-kernels *************************/
Marat Dukhan	66a3ca1	2021-08-06 18:24:19 -0700	[diff] [blame]	731	#ifndef XNN_NO_QS8_OPERATORS
				732	init_flags \|= XNN_INIT_FLAG_QS8;
				733
				734	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_2x2__scalar_magic);
				735	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_2x2__scalar_magic);
				736	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x2__scalar_magic);
				737	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x2__scalar_magic);
				738	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_scalar_magic_params;
				739	xnn_params.qs8.gemm.mr = 2;
				740	xnn_params.qs8.gemm.nr = 2;
				741
				742	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up1x9__scalar_magic;
				743	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_scalar_magic_params;
				744	xnn_params.qs8.dwconv[0].channel_tile = 1;
				745	xnn_params.qs8.dwconv[0].primary_tile = 9;
				746	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up1x25__scalar_magic;
				747	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_scalar_magic_params;
				748	xnn_params.qs8.dwconv[1].channel_tile = 1;
				749	xnn_params.qs8.dwconv[1].primary_tile = 25;
				750
				751	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				752	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__scalar_c1,
				753	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
				754	.mr = 7,
				755	};
				756	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				757	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__scalar_x1,
				758	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__scalar_x1,
				759	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__scalar_x1,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	760	.init.qs8_addsub = xnn_init_qs8_add_minmax_scalar_params,
Marat Dukhan	66a3ca1	2021-08-06 18:24:19 -0700	[diff] [blame]	761	.element_tile = 1,
				762	};
				763	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				764	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__scalar_x4,
				765	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__scalar_x4,
				766	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__scalar_x4,
				767	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_scalar_params,
				768	.element_tile = 4,
				769	};
				770	#endif // XNN_NO_QS8_OPERATORS
				771
				772	/************************* QU8 micro-kernels *************************/
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	773	#ifndef XNN_NO_QU8_OPERATORS
				774	init_flags \|= XNN_INIT_FLAG_QU8;
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	775
Marat Dukhan	43b46ee	2021-07-15 19:07:50 -0700	[diff] [blame]	776	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_2x2__scalar_magic);
				777	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_2x2__scalar_magic);
				778	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x2__scalar_magic);
				779	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x2__scalar_magic);
				780	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_scalar_magic_params;
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	781	xnn_params.qu8.gemm.mr = 2;
				782	xnn_params.qu8.gemm.nr = 2;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	783
Marat Dukhan	43b46ee	2021-07-15 19:07:50 -0700	[diff] [blame]	784	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up1x9__scalar_magic;
				785	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_scalar_magic_params;
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	786	xnn_params.qu8.dwconv[0].channel_tile = 1;
				787	xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhan	43b46ee	2021-07-15 19:07:50 -0700	[diff] [blame]	788	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up1x25__scalar_magic;
				789	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_scalar_magic_params;
				790	xnn_params.qu8.dwconv[1].channel_tile = 1;
				791	xnn_params.qu8.dwconv[1].primary_tile = 25;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	792
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	793	xnn_params.qu8.avgpool = (struct avgpool_parameters) {
				794	.up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__scalar_c1,
				795	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	796	.mr = 9,
				797	.qr = 8,
				798	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	799	xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
				800	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__scalar_c1,
				801	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	802	.mr = 7,
				803	};
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	804	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				805	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__scalar_x1,
				806	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__scalar_x1,
				807	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__scalar_x1,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	808	.init.qu8_addsub = xnn_init_qu8_add_minmax_scalar_params,
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	809	.element_tile = 1,
				810	};
Marat Dukhan	3c5e662	2021-08-06 00:38:05 -0700	[diff] [blame]	811	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				812	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__scalar_x4,
				813	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__scalar_x4,
				814	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__scalar_x4,
				815	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_scalar_params,
				816	.element_tile = 4,
				817	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	818	#endif // XNN_NO_QU8_OPERATORS
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	819
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	820	/************************** S8 micro-kernels **************************/
				821	#ifndef XNN_NO_S8_OPERATORS
				822	init_flags \|= XNN_INIT_FLAG_S8;
				823
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	824	xnn_params.s8.clamp = (struct vunary_parameters) {
				825	.ukernel = (xnn_univector_ukernel_function) xnn_s8_vclamp_ukernel__scalar_x4,
Marat Dukhan	3b9b4bc	2021-08-24 00:23:07 -0700	[diff] [blame]	826	.init.s8_minmax = xnn_init_s8_minmax_scalar_params,
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	827	.element_tile = 4,
				828	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	829	xnn_params.s8.ibilinear = (struct ibilinear_parameters) {
				830	.ukernel = (xnn_ibilinear_ukernel_function) xnn_s8_ibilinear_ukernel__scalar_c1,
				831	.pixel_tile = 1,
				832	.channel_tile = 1,
				833	};
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	834	xnn_params.s8.maxpool = (struct maxpool_parameters) {
				835	.ukernel = (xnn_maxpool_ukernel_function) xnn_s8_maxpool_minmax_ukernel_9p8x__scalar_c1,
				836	.init.s8 = xnn_init_s8_minmax_scalar_params,
				837	.mr = 9,
				838	.qr = 8,
				839	};
				840	#endif // XNN_NO_S8_OPERATORS
				841
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	842	/************************** U8 micro-kernels **************************/
				843	#ifndef XNN_NO_U8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	844	init_flags \|= XNN_INIT_FLAG_U8;
				845
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	846	xnn_params.u8.clamp = (struct vunary_parameters) {
				847	.ukernel = (xnn_univector_ukernel_function) xnn_u8_vclamp_ukernel__scalar_x4,
Marat Dukhan	3b9b4bc	2021-08-24 00:23:07 -0700	[diff] [blame]	848	.init.u8_minmax = xnn_init_u8_minmax_scalar_params,
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	849	.element_tile = 4,
				850	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	851	xnn_params.u8.ibilinear = (struct ibilinear_parameters) {
				852	.ukernel = (xnn_ibilinear_ukernel_function) xnn_u8_ibilinear_ukernel__scalar_c1,
				853	.pixel_tile = 1,
				854	.channel_tile = 1,
				855	};
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	856	xnn_params.u8.maxpool = (struct maxpool_parameters) {
				857	.ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	858	.init.u8 = xnn_init_u8_minmax_scalar_params,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	859	.mr = 9,
				860	.qr = 8,
				861	};
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	862	xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
				863	xnn_params.u8.rmax = xnn_u8_rmax_ukernel__scalar;
				864	#endif // XNN_NO_U8_OPERATORS
				865
				866	/************************** X8 micro-kernels **************************/
				867	#ifndef XNN_NO_X8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	868	init_flags \|= XNN_INIT_FLAG_X8;
				869
Marat Dukhan	d67539d	2021-09-08 23:06:03 -0700	[diff] [blame]	870	xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar_x4;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	871	xnn_params.x8.zip = (struct zip_parameters) {
				872	.x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__scalar,
				873	.x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__scalar,
				874	.x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__scalar,
				875	.xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__scalar,
				876	};
				877	#endif // XNN_NO_X8_OPERATORS
				878
				879	/************************** F32 micro-kernels **************************/
				880	#ifndef XNN_NO_F32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	881	init_flags \|= XNN_INIT_FLAG_F32;
				882
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	883	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x4__scalar);
				884	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x4__scalar);
				885	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x4__scalar);
				886	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x4__scalar);
Marat Dukhan	467f636	2020-05-22 23:21:55 -0700	[diff] [blame]	887	xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_4x4__scalar);
				888	xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_4x4__scalar);
				889	xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x4__scalar);
				890	xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x4__scalar);
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	891	xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x4__scalar);
				892	xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x4__scalar);
				893	xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x4__scalar);
				894	xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x4__scalar);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	895	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	896	xnn_params.f32.gemm.mr = 4;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	897	xnn_params.f32.gemm.nr = 4;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	898
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	899	xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__scalar);
				900	xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__scalar),
				901	xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2__scalar);
				902	xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2__scalar),
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	903	xnn_params.f32.gemm2.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	904	xnn_params.f32.gemm2.mr = 4;
				905	xnn_params.f32.gemm2.nr = 2;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	906
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	907	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x3__scalar_acc2;
				908	xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x3__scalar_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	909	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	910	xnn_params.f32.dwconv[0].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	911	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	912
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	913	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x4__scalar_acc2;
				914	xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x4__scalar_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	915	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	916	xnn_params.f32.dwconv[1].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	917	xnn_params.f32.dwconv[1].primary_tile = 4;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	918
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	919	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x9__scalar_acc2;
				920	xnn_params.f32.dwconv[2].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x9__scalar_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	921	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	922	xnn_params.f32.dwconv[2].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	923	xnn_params.f32.dwconv[2].primary_tile = 9;
				924
				925	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x25__scalar_acc2;
				926	xnn_params.f32.dwconv[3].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x25__scalar_acc2;
				927	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_scalar_params;
				928	xnn_params.f32.dwconv[3].channel_tile = 1;
				929	xnn_params.f32.dwconv[3].primary_tile = 25;
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	930
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	931	xnn_params.f32.avgpool = (struct avgpool_parameters) {
				932	.up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__scalar_c1,
				933	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__scalar_c1,
				934	.mr = 9,
				935	.qr = 8,
				936	};
				937	xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
				938	.up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__scalar_c1,
				939	.mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__scalar_c1,
				940	.mr = 9,
				941	.qr = 8,
				942	};
				943	xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
				944	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__scalar_c1,
				945	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__scalar_c1,
				946	.mr = 7,
				947	};
				948	xnn_params.f32.maxpool = (struct maxpool_parameters) {
				949	.ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	950	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	951	.mr = 9,
				952	.qr = 8,
				953	};
				954	xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
				955	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__scalar_c1,
				956	.mr = 4,
				957	};
				958	xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
				959	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__scalar_c1,
				960	.mr = 9,
				961	};
				962	xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
				963	.mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__scalar_c1,
				964	.mr = 9,
				965	.qr = 8,
				966	};
				967	xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
				968	.ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__scalar_c2,
				969	.pixel_tile = 1,
				970	.channel_tile = 2,
				971	};
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	972	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__scalar_x4;
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	973	xnn_params.f32.clamp = (struct vunary_parameters) {
				974	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__scalar_x4,
				975	.init.f32_minmax = xnn_init_f32_minmax_scalar_params,
				976	.element_tile = 4,
				977	};
Marat Dukhan	b6bd4bc	2020-12-01 17:01:40 -0800	[diff] [blame]	978	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__scalar_rr2_lut16_p3_x4;
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	979	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__scalar_x4;
Marat Dukhan	2881333	2020-06-10 18:05:38 -0700	[diff] [blame]	980	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__scalar_x4;
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	981	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__scalar_x4;
Marat Dukhan	64e5251	2020-06-09 13:41:16 -0700	[diff] [blame]	982	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__scalar_libm_x1;
				983	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__scalar_libm_x1;
				984	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__scalar_libm_x1;
				985	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__scalar_libm_x1;
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	986	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__scalar_lut64_p2_div_x2;
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	987	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__scalar_x4;
Marat Dukhan	6804bbd	2020-06-30 19:26:11 -0700	[diff] [blame]	988	xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__scalar_sqrt_x1;
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	989	xnn_params.f32.prelu = (struct prelu_parameters) {
				990	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__scalar_2x4,
				991	.row_tile = 4,
				992	.channel_tile = 4,
				993	};
				994	xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__scalar_p5_x4_acc2;
				995	xnn_params.f32.rmax = xnn_f32_rmax_ukernel__scalar;
				996	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	997	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__scalar_x8,
				998	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__scalar_x8,
				999	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__scalar_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1000	.element_tile = 8,
				1001	};
				1002	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	1003	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__scalar_x2,
				1004	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__scalar_x2,
				1005	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__scalar_x2,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1006	.element_tile = 2,
				1007	};
				1008	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	1009	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__scalar_x8,
				1010	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__scalar_x8,
				1011	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__scalar_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1012	.element_tile = 8,
				1013	};
				1014	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	1015	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__scalar_x8,
				1016	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__scalar_x8,
				1017	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__scalar_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1018	.element_tile = 8,
				1019	};
				1020	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	1021	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__scalar_x8,
				1022	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__scalar_x8,
				1023	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__scalar_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1024	.element_tile = 8,
				1025	};
				1026	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	1027	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__scalar_x8,
				1028	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__scalar_x8,
				1029	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__scalar_x8,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1030	.element_tile = 8,
				1031	};
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	1032	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	1033	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__scalar_x8,
				1034	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
				1035	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	1036	.element_tile = 8,
				1037	};
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1038	xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan	9531e9f	2020-07-24 15:25:02 -0700	[diff] [blame]	1039	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c1__scalar_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	1040	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1041	.channel_tile = 1,
				1042	.row_tile = 2,
				1043	};
				1044	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	cfbed0a	2020-12-08 10:01:51 -0800	[diff] [blame]	1045	init_flags \|= XNN_INIT_FLAG_CHW_OPT;
				1046
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1047	xnn_params.f32.spmm = (struct spmm_parameters) {
				1048	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x1__scalar,
				1049	.mr = 8,
				1050	.nr = 1,
				1051	};
				1052	xnn_params.f32.spmm2 = (struct spmm_parameters) {
				1053	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x2__scalar,
				1054	.mr = 8,
				1055	.nr = 2,
				1056	};
				1057	xnn_params.f32.spmm4 = (struct spmm_parameters) {
				1058	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x4__scalar,
				1059	.mr = 8,
				1060	.nr = 4,
				1061	};
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	1062	xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1063	.ukernel_with_symm_padding =
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	1064	(xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__scalar_1x1,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1065	.output_channel_tile = 4,
				1066	.output_height_tile = 1,
				1067	.output_width_tile = 1,
				1068	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	1069	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
Marat Dukhan	4ddfab4	2020-12-07 17:33:11 -0800	[diff] [blame]	1070	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__scalar_4x1,
Marat Dukhan	4ddfab4	2020-12-07 17:33:11 -0800	[diff] [blame]	1071	.output_width_tile = 1,
				1072	.output_height_tile = 4,
				1073	};
				1074	xnn_params.f32.dwconv2d_chw_3x3s2 = (struct dwconv2d_chw_parameters) {
				1075	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__scalar_2x1_acc2,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1076	.output_width_tile = 1,
Marat Dukhan	91249d2	2020-10-24 12:02:51 -0700	[diff] [blame]	1077	.output_height_tile = 2,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1078	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	1079	xnn_params.f32.dwconv2d_chw_5x5 = (struct dwconv2d_chw_parameters) {
Marat Dukhan	4ddfab4	2020-12-07 17:33:11 -0800	[diff] [blame]	1080	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5p2__scalar_2x1_acc2,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1081	.output_width_tile = 1,
Marat Dukhan	4ddfab4	2020-12-07 17:33:11 -0800	[diff] [blame]	1082	.output_height_tile = 2,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1083	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	1084	xnn_params.f32.dwconv2d_chw_5x5s2 = (struct dwconv2d_chw_parameters) {
Marat Dukhan	4ddfab4	2020-12-07 17:33:11 -0800	[diff] [blame]	1085	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__scalar_2x1_acc2,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1086	.output_width_tile = 1,
Marat Dukhan	4ddfab4	2020-12-07 17:33:11 -0800	[diff] [blame]	1087	.output_height_tile = 2,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1088	};
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	1089	xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
				1090	.ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__scalar_x1,
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1091	.channel_tile = 1,
				1092	};
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	1093	xnn_params.f32.ibilinear_chw = (struct ibilinear_chw_parameters) {
				1094	.ukernel = (xnn_ibilinear_chw_ukernel_function) xnn_f32_ibilinear_chw_ukernel__scalar_p4,
				1095	.channel_tile = 1,
				1096	.pixel_tile = 4,
				1097	};
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1098	#endif // XNN_NO_NCHW_OPERATORS
				1099	#endif // XNN_NO_F32_OPERATORS
				1100
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	1101	/************************* VCVT micro-kernels *************************/
				1102	#ifndef XNN_NO_VCVT_OPERATORS
				1103	init_flags \|= XNN_INIT_FLAG_VCVT;
				1104
				1105	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__scalar_float_x4;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	1106	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__scalar_fabsf_x2;
Marat Dukhan	ed2d776	2021-12-03 23:51:19 -0800	[diff] [blame]	1107	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
Marat Dukhan	430b173	2021-12-04 02:53:12 -0800	[diff] [blame]	1108	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qs8_vcvt_ukernel__scalar_magic_iminmax_x4,
				1109	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_scalar_magic_iminmax_params,
Marat Dukhan	ed2d776	2021-12-03 23:51:19 -0800	[diff] [blame]	1110	.element_tile = 4,
				1111	};
				1112	xnn_params.vcvt.f32_to_qu8 = (struct vunary_parameters) {
Marat Dukhan	430b173	2021-12-04 02:53:12 -0800	[diff] [blame]	1113	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__scalar_magic_iminmax_x4,
				1114	.init.f32_qu8_cvt = xnn_init_f32_qu8_cvt_scalar_magic_iminmax_params,
Marat Dukhan	ed2d776	2021-12-03 23:51:19 -0800	[diff] [blame]	1115	.element_tile = 4,
				1116	};
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	1117	#endif // XNN_NO_VCVT_OPERATORS
				1118
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1119	/************************** X32 micro-kernels **************************/
				1120	#ifndef XNN_NO_X32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	1121	init_flags \|= XNN_INIT_FLAG_X32;
				1122
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1123	xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__scalar;
				1124	xnn_params.x32.zip = (struct zip_parameters) {
				1125	.x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__scalar,
				1126	.x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__scalar,
				1127	.x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__scalar,
				1128	.xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__scalar,
				1129	};
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	1130	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	ad71b9a	2020-11-20 00:01:51 -0800	[diff] [blame]	1131	xnn_params.x32.depthtospace2d_chw2hwc = (struct depthtospace2d_chw2hwc_parameters) {
				1132	.ukernel = (xnn_depthtospace2d_chw2hwc_ukernel_function) xnn_x32_depthtospace2d_chw2hwc_ukernel__scalar,
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	1133	.channel_tile = 1,
				1134	.pixel_tile = 1,
				1135	};
				1136	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1137	#endif // XNN_NO_X32_OPERATORS
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	1138
				1139	/************************** XX micro-kernels **************************/
				1140	#ifndef XNN_NO_XX_OPERATORS
				1141	init_flags \|= XNN_INIT_FLAG_XX;
				1142
				1143	xnn_params.xx.copy = (xnn_univector_ukernel_function) xnn_xx_copy_ukernel__memcpy;
				1144	xnn_params.xx.fill = (struct fill_parameters) {
				1145	.ukernel = (xnn_fill_ukernel_function) xnn_xx_fill_ukernel__scalar_x16,
				1146	.row_tile = 1,
				1147	};
Marat Dukhan	0461f2d	2021-08-08 12:36:29 -0700	[diff] [blame]	1148	xnn_params.xx.pad = (struct pad_parameters) {
				1149	.ukernel = (xnn_pad_ukernel_function) xnn_xx_pad_ukernel__scalar,
				1150	.row_tile = 1,
				1151	};
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	1152	#endif // XNN_NO_XX_OPERATORS
Marat Dukhan	3b745a4	2020-05-10 21:43:25 -0700	[diff] [blame]	1153	}
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1154
Marat Dukhan	1dadbf7	2019-10-01 10:46:20 -0700	[diff] [blame]	1155	#elif XNN_ARCH_ARM64
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1156
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	1157	/************************** QC8 micro-kernels **************************/
				1158	#ifndef XNN_NO_QC8_OPERATORS
				1159	init_flags \|= XNN_INIT_FLAG_QC8;
				1160
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1161	#if XNN_PLATFORM_IOS \|\| XNN_PLATFORM_MAC
				1162	#if XNN_ENABLE_ASSEMBLY
				1163	if (cpuinfo_has_arm_neon_dot()) {
				1164	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_ld128);
				1165	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x16c4__neondot);
				1166	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_ld128);
				1167	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x16c4__neondot);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1168	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1169	xnn_params.qc8.gemm.mr = 4;
				1170	xnn_params.qc8.gemm.nr = 16;
				1171	xnn_params.qc8.gemm.log2_kr = 2;
				1172	} else {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1173	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal);
				1174	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal);
				1175	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c8__aarch64_neon_mlal);
				1176	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c8__aarch64_neon_mlal);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1177	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1178	xnn_params.qc8.gemm.mr = 2;
				1179	xnn_params.qc8.gemm.nr = 8;
				1180	xnn_params.qc8.gemm.log2_kr = 3;
				1181	}
				1182	#else // !XNN_ENABLE_ASSEMBLY
				1183	if (cpuinfo_has_arm_neon_dot()) {
				1184	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16c4__neondot);
				1185	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x16c4__neondot);
				1186	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16c4__neondot);
				1187	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x16c4__neondot);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1188	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1189	xnn_params.qc8.gemm.mr = 4;
				1190	xnn_params.qc8.gemm.nr = 16;
				1191	xnn_params.qc8.gemm.log2_kr = 2;
				1192	} else {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1193	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x8c2s4__neonv8_mlal);
				1194	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x8c2s4__neonv8_mlal);
				1195	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c2s4__neonv8_mlal);
				1196	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c2s4__neonv8_mlal);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1197	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1198	xnn_params.qc8.gemm.mr = 2;
				1199	xnn_params.qc8.gemm.nr = 8;
				1200	xnn_params.qc8.gemm.log2_kr = 1;
Frank Barchard	e7043ff	2021-11-10 14:50:08 -0800	[diff] [blame]	1201	xnn_params.qc8.gemm.log2_sr = 2;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1202	}
				1203	#endif // XNN_ENABLE_ASSEMBLY
				1204	#else // !XNN_PLATFORM_IOS && !XNN_PLATFORM_MAC
				1205	#if XNN_ENABLE_ASSEMBLY
				1206	if (cpuinfo_has_arm_neon_dot()) {
				1207	switch (cpuinfo_get_core(0)->uarch) {
				1208	case cpuinfo_uarch_cortex_a55:
				1209	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_cortex_a55);
				1210	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_cortex_a55);
				1211	break;
				1212	case cpuinfo_uarch_cortex_x1:
				1213	case cpuinfo_uarch_cortex_a78:
				1214	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_ld128);
				1215	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_ld128);
				1216	break;
				1217	default:
				1218	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_ld64);
				1219	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_ld64);
				1220	break;
				1221	}
				1222	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x16c4__neondot);
				1223	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x16c4__neondot);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1224	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1225	xnn_params.qc8.gemm.mr = 4;
				1226	xnn_params.qc8.gemm.nr = 16;
				1227	xnn_params.qc8.gemm.log2_kr = 2;
				1228	} else {
				1229	switch (cpuinfo_get_core(0)->uarch) {
Frank Barchard	6c34dbf	2021-11-22 16:14:53 -0800	[diff] [blame]	1230	case cpuinfo_uarch_cortex_a35:
				1231	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_ld64);
				1232	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_ld64);
				1233	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x16__neonv8_mlal_lane);
				1234	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x16__neonv8_mlal_lane);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1235	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Frank Barchard	6c34dbf	2021-11-22 16:14:53 -0800	[diff] [blame]	1236	xnn_params.qc8.gemm.mr = 4;
				1237	xnn_params.qc8.gemm.nr = 16;
				1238	break;
				1239
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1240	case cpuinfo_uarch_cortex_a53:
				1241	case cpuinfo_uarch_cortex_a55r0:
				1242	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53);
				1243	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53);
				1244	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x16__neonv8_mlal_lane);
				1245	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x16__neonv8_mlal_lane);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1246	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1247	xnn_params.qc8.gemm.mr = 4;
				1248	xnn_params.qc8.gemm.nr = 16;
				1249	break;
				1250
				1251	case cpuinfo_uarch_cortex_a72:
				1252	case cpuinfo_uarch_cortex_a73:
				1253	case cpuinfo_uarch_kryo:
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1254	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal_prfm);
				1255	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal_prfm);
				1256	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c8__aarch64_neon_mlal_prfm);
				1257	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c8__aarch64_neon_mlal_prfm);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1258	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1259	xnn_params.qc8.gemm.mr = 2;
				1260	xnn_params.qc8.gemm.nr = 8;
				1261	xnn_params.qc8.gemm.log2_kr = 3;
				1262	break;
				1263
				1264	default:
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1265	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal);
				1266	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal);
				1267	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c8__aarch64_neon_mlal);
				1268	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c8__aarch64_neon_mlal);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1269	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1270	xnn_params.qc8.gemm.mr = 2;
				1271	xnn_params.qc8.gemm.nr = 8;
				1272	xnn_params.qc8.gemm.log2_kr = 3;
				1273	break;
				1274	}
				1275	}
				1276	#if XNN_MAX_UARCH_TYPES > 1
				1277	{
				1278	/* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
				1279	const uint32_t mr = xnn_params.qc8.gemm.mr;
				1280	const uint32_t nr = xnn_params.qc8.gemm.nr;
				1281	const uint32_t log2_kr = xnn_params.qc8.gemm.log2_kr;
				1282	for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
				1283	const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
				1284	if (uarch_info == NULL) {
				1285	/* No more microarchitectures in the system */
				1286	break;
				1287	}
				1288
				1289	switch (uarch_info->uarch) {
				1290	case cpuinfo_uarch_cortex_a53:
				1291	case cpuinfo_uarch_cortex_a55r0:
				1292	if (mr == 2 && nr == 8 && log2_kr == 3) {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1293	xnn_params.qc8.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal_prfm_cortex_a53;
				1294	xnn_params.qc8.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal_prfm_cortex_a53;
				1295	xnn_params.qc8.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c8__aarch64_neon_mlal_prfm_cortex_a53;
				1296	xnn_params.qc8.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c8__aarch64_neon_mlal_prfm_cortex_a53;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1297	}
				1298	break;
				1299
				1300	case cpuinfo_uarch_cortex_a55:
Frank Barchard	c37b8da	2021-09-01 00:35:19 -0700	[diff] [blame]	1301	if (mr == 4 && nr == 16 && log2_kr == 2 && cpuinfo_has_arm_neon_dot()) {
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1302	xnn_params.qc8.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_cortex_a55;
				1303	xnn_params.qc8.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_cortex_a55;
				1304	xnn_params.qc8.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x16c4__neondot;
				1305	xnn_params.qc8.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x16c4__neondot;
				1306	}
				1307	break;
				1308	default:
				1309	break;
				1310	}
				1311	}
				1312	}
				1313	#endif // XNN_MAX_UARCH_TYPES > 1
				1314	#else // !XNN_ENABLE_ASSEMBLY
				1315	if (cpuinfo_has_arm_neon_dot()) {
				1316	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16c4__neondot);
				1317	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x16c4__neondot);
				1318	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16c4__neondot);
				1319	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x16c4__neondot);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1320	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1321	xnn_params.qc8.gemm.mr = 4;
				1322	xnn_params.qc8.gemm.nr = 16;
				1323	xnn_params.qc8.gemm.log2_kr = 2;
				1324	} else {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1325	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x8c2s4__neonv8_mlal);
				1326	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x8c2s4__neonv8_mlal);
				1327	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c2s4__neonv8_mlal);
				1328	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c2s4__neonv8_mlal);
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1329	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1330	xnn_params.qc8.gemm.mr = 2;
				1331	xnn_params.qc8.gemm.nr = 8;
				1332	xnn_params.qc8.gemm.log2_kr = 1;
Frank Barchard	e7043ff	2021-11-10 14:50:08 -0800	[diff] [blame]	1333	xnn_params.qc8.gemm.log2_sr = 2;
Marat Dukhan	75d1b79	2021-07-01 13:00:28 -0700	[diff] [blame]	1334	}
				1335	#endif // XNN_ENABLE_ASSEMBLY
				1336	#endif // XNN_PLATFORM_IOS \|\| XNN_PLATFORM_MAC
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	1337
Frank Barchard	0d06573	2021-08-31 00:01:40 -0700	[diff] [blame]	1338	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8_ld64;
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1339	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Frank Barchard	0d06573	2021-08-31 00:01:40 -0700	[diff] [blame]	1340	xnn_params.qc8.dwconv[0].channel_tile = 16;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	1341	xnn_params.qc8.dwconv[0].primary_tile = 9;
Frank Barchard	7da8b02	2021-08-31 09:49:10 -0700	[diff] [blame]	1342	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8_ld64;
Marat Dukhan	7988a18	2021-12-06 22:00:33 -0800	[diff] [blame]	1343	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_neonv8_params;
Frank Barchard	7da8b02	2021-08-31 09:49:10 -0700	[diff] [blame]	1344	xnn_params.qc8.dwconv[1].channel_tile = 16;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	1345	xnn_params.qc8.dwconv[1].primary_tile = 25;
				1346	#endif // XNN_NO_QC8_OPERATORS
				1347
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	1348	/************************** QS8 micro-kernels **************************/
				1349	#ifndef XNN_NO_QS8_OPERATORS
				1350	init_flags \|= XNN_INIT_FLAG_QS8;
				1351
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	1352	#if XNN_PLATFORM_IOS \|\| XNN_PLATFORM_MAC
Frank Barchard	bc0c729	2020-10-06 13:36:54 -0700	[diff] [blame]	1353	#if XNN_ENABLE_ASSEMBLY
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1354	if (cpuinfo_has_arm_neon_dot()) {
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1355	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_ld128);
				1356	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1357	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_ld128);
				1358	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1359	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1360	xnn_params.qs8.gemm.mr = 4;
				1361	xnn_params.qs8.gemm.nr = 16;
				1362	xnn_params.qs8.gemm.log2_kr = 2;
				1363	} else {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1364	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_2x8c8__aarch64_neon_mlal);
				1365	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_2x8c8__aarch64_neon_mlal);
				1366	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x8c8__aarch64_neon_mlal);
				1367	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x8c8__aarch64_neon_mlal);
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1368	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1369	xnn_params.qs8.gemm.mr = 2;
Frank Barchard	a414daa	2021-02-23 15:50:39 -0800	[diff] [blame]	1370	xnn_params.qs8.gemm.nr = 8;
Frank Barchard	bbf5182	2021-03-12 10:37:31 -0800	[diff] [blame]	1371	xnn_params.qs8.gemm.log2_kr = 3;
Frank Barchard	1e8590e	2020-10-12 21:20:46 -0700	[diff] [blame]	1372	}
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1373	#else // !XNN_ENABLE_ASSEMBLY
				1374	if (cpuinfo_has_arm_neon_dot()) {
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1375	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c4__neondot);
				1376	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1377	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16c4__neondot);
				1378	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1379	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1380	xnn_params.qs8.gemm.mr = 4;
				1381	xnn_params.qs8.gemm.nr = 16;
				1382	xnn_params.qs8.gemm.log2_kr = 2;
				1383	} else {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1384	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_2x8c2s4__neon_mlal);
				1385	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_2x8c2s4__neon_mlal);
				1386	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x8c2s4__neon_mlal);
				1387	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x8c2s4__neon_mlal);
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1388	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1389	xnn_params.qs8.gemm.mr = 2;
Frank Barchard	a414daa	2021-02-23 15:50:39 -0800	[diff] [blame]	1390	xnn_params.qs8.gemm.nr = 8;
				1391	xnn_params.qs8.gemm.log2_kr = 1;
Frank Barchard	66ae257	2021-11-02 17:36:21 -0700	[diff] [blame]	1392	xnn_params.qs8.gemm.log2_sr = 2;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1393	}
				1394	#endif // XNN_ENABLE_ASSEMBLY
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	1395	#else // !XNN_PLATFORM_IOS && !XNN_PLATFORM_MAC
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1396	#if XNN_ENABLE_ASSEMBLY
				1397	if (cpuinfo_has_arm_neon_dot()) {
				1398	switch (cpuinfo_get_core(0)->uarch) {
				1399	case cpuinfo_uarch_cortex_a55:
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1400	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_cortex_a55);
				1401	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_cortex_a55);
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1402	break;
Frank Barchard	0ae35f2	2021-06-15 17:34:24 -0700	[diff] [blame]	1403	case cpuinfo_uarch_cortex_x1:
				1404	case cpuinfo_uarch_cortex_a78:
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1405	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_ld128);
				1406	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_ld128);
Frank Barchard	0ae35f2	2021-06-15 17:34:24 -0700	[diff] [blame]	1407	break;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1408	default:
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1409	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_ld64);
				1410	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_ld64);
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1411	break;
				1412	}
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1413	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1414	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1415	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1416	xnn_params.qs8.gemm.mr = 4;
				1417	xnn_params.qs8.gemm.nr = 16;
				1418	xnn_params.qs8.gemm.log2_kr = 2;
				1419	} else {
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1420	switch (cpuinfo_get_core(0)->uarch) {
Frank Barchard	6c34dbf	2021-11-22 16:14:53 -0800	[diff] [blame]	1421	case cpuinfo_uarch_cortex_a35:
				1422	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_ld64);
				1423	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_ld64);
				1424	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1425	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1426	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
				1427	xnn_params.qs8.gemm.mr = 4;
				1428	xnn_params.qs8.gemm.nr = 16;
				1429	break;
				1430
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1431	case cpuinfo_uarch_cortex_a53:
Frank Barchard	fb5983d	2021-04-20 14:09:08 -0700	[diff] [blame]	1432	case cpuinfo_uarch_cortex_a55r0:
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1433	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53);
				1434	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53);
				1435	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1436	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1437	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Frank Barchard	d4416d6	2021-05-17 15:51:37 -0700	[diff] [blame]	1438	xnn_params.qs8.gemm.mr = 4;
				1439	xnn_params.qs8.gemm.nr = 16;
Frank Barchard	6ac1d18	2021-04-14 13:47:07 -0700	[diff] [blame]	1440	break;
				1441
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1442	case cpuinfo_uarch_cortex_a72:
				1443	case cpuinfo_uarch_cortex_a73:
				1444	case cpuinfo_uarch_kryo:
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1445	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_2x8c8__aarch64_neon_mlal_prfm);
				1446	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_2x8c8__aarch64_neon_mlal_prfm);
				1447	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x8c8__aarch64_neon_mlal_prfm);
				1448	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x8c8__aarch64_neon_mlal_prfm);
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1449	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1450	xnn_params.qs8.gemm.mr = 2;
				1451	xnn_params.qs8.gemm.nr = 8;
				1452	xnn_params.qs8.gemm.log2_kr = 3;
Frank Barchard	c77fc4c	2021-04-14 13:28:01 -0700	[diff] [blame]	1453	break;
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1454
				1455	default:
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1456	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_2x8c8__aarch64_neon_mlal);
				1457	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_2x8c8__aarch64_neon_mlal);
				1458	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x8c8__aarch64_neon_mlal);
				1459	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x8c8__aarch64_neon_mlal);
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1460	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1461	xnn_params.qs8.gemm.mr = 2;
				1462	xnn_params.qs8.gemm.nr = 8;
				1463	xnn_params.qs8.gemm.log2_kr = 3;
Frank Barchard	c77fc4c	2021-04-14 13:28:01 -0700	[diff] [blame]	1464	break;
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1465	}
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1466	}
				1467	#if XNN_MAX_UARCH_TYPES > 1
				1468	{
				1469	/* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
				1470	const uint32_t mr = xnn_params.qs8.gemm.mr;
				1471	const uint32_t nr = xnn_params.qs8.gemm.nr;
				1472	const uint32_t log2_kr = xnn_params.qs8.gemm.log2_kr;
				1473	for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
				1474	const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
				1475	if (uarch_info == NULL) {
				1476	/* No more microarchitectures in the system */
				1477	break;
				1478	}
				1479
				1480	switch (uarch_info->uarch) {
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1481	case cpuinfo_uarch_cortex_a53:
Frank Barchard	90f520b	2021-04-26 18:01:51 -0700	[diff] [blame]	1482	case cpuinfo_uarch_cortex_a55r0:
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1483	if (mr == 2 && nr == 8 && log2_kr == 3) {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1484	xnn_params.qs8.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_2x8c8__aarch64_neon_mlal_prfm_cortex_a53;
				1485	xnn_params.qs8.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_2x8c8__aarch64_neon_mlal_prfm_cortex_a53;
				1486	xnn_params.qs8.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x8c8__aarch64_neon_mlal_prfm_cortex_a53;
				1487	xnn_params.qs8.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x8c8__aarch64_neon_mlal_prfm_cortex_a53;
Frank Barchard	2a995e7	2021-04-13 16:24:25 -0700	[diff] [blame]	1488	}
				1489	break;
				1490
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1491	case cpuinfo_uarch_cortex_a55:
Frank Barchard	c37b8da	2021-09-01 00:35:19 -0700	[diff] [blame]	1492	if (mr == 4 && nr == 16 && log2_kr == 2 && cpuinfo_has_arm_neon_dot()) {
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1493	xnn_params.qs8.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_cortex_a55;
				1494	xnn_params.qs8.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_cortex_a55;
				1495	xnn_params.qs8.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x16c4__neondot;
				1496	xnn_params.qs8.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x16c4__neondot;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1497	}
				1498	break;
				1499	default:
				1500	break;
				1501	}
				1502	}
				1503	}
				1504	#endif // XNN_MAX_UARCH_TYPES > 1
				1505	#else // !XNN_ENABLE_ASSEMBLY
				1506	if (cpuinfo_has_arm_neon_dot()) {
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1507	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c4__neondot);
				1508	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1509	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_4x16c4__neondot);
				1510	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1511	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1512	xnn_params.qs8.gemm.mr = 4;
				1513	xnn_params.qs8.gemm.nr = 16;
				1514	xnn_params.qs8.gemm.log2_kr = 2;
				1515	} else {
Frank Barchard	e22685a	2021-11-12 11:36:58 -0800	[diff] [blame]	1516	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_2x8c2s4__neon_mlal);
				1517	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_2x8c2s4__neon_mlal);
				1518	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x8c2s4__neon_mlal);
				1519	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x8c2s4__neon_mlal);
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	1520	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1521	xnn_params.qs8.gemm.mr = 2;
Frank Barchard	a414daa	2021-02-23 15:50:39 -0800	[diff] [blame]	1522	xnn_params.qs8.gemm.nr = 8;
				1523	xnn_params.qs8.gemm.log2_kr = 1;
Frank Barchard	e7043ff	2021-11-10 14:50:08 -0800	[diff] [blame]	1524	xnn_params.qs8.gemm.log2_sr = 2;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	1525	}
				1526	#endif // XNN_ENABLE_ASSEMBLY
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	1527	#endif // XNN_PLATFORM_IOS \|\| XNN_PLATFORM_MAC
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	1528
Frank Barchard	0d06573	2021-08-31 00:01:40 -0700	[diff] [blame]	1529	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8_ld64;
Marat Dukhan	4ba70b7	2021-07-19 11:20:16 -0700	[diff] [blame]	1530	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Frank Barchard	0d06573	2021-08-31 00:01:40 -0700	[diff] [blame]	1531	xnn_params.qs8.dwconv[0].channel_tile = 16;
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	1532	xnn_params.qs8.dwconv[0].primary_tile = 9;
Frank Barchard	7da8b02	2021-08-31 09:49:10 -0700	[diff] [blame]	1533	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8_ld64;
Marat Dukhan	4ba70b7	2021-07-19 11:20:16 -0700	[diff] [blame]	1534	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
Frank Barchard	7da8b02	2021-08-31 09:49:10 -0700	[diff] [blame]	1535	xnn_params.qs8.dwconv[1].channel_tile = 16;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	1536	xnn_params.qs8.dwconv[1].primary_tile = 25;
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	1537
				1538	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				1539	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__neon_c8_acc2,
				1540	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__neon_c8_acc2,
				1541	.mr = 7,
				1542	};
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	1543
				1544	xnn_params.qs8.vadd = (struct vbinary_parameters) {
Marat Dukhan	01debd9	2021-07-29 18:14:21 -0700	[diff] [blame]	1545	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__neon_ld64_x32,
				1546	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x32,
				1547	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x32,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	1548	.init.qs8_addsub = xnn_init_qs8_add_minmax_neon_params,
Marat Dukhan	01debd9	2021-07-29 18:14:21 -0700	[diff] [blame]	1549	.element_tile = 32,
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	1550	};
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	1551	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				1552	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__neonv8_ld64_x16,
				1553	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__neonv8_ld64_x16,
				1554	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__neonv8_ld64_x16,
				1555	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_neonv8_params,
				1556	.element_tile = 16,
				1557	};
Marat Dukhan	f28cddf	2020-08-10 14:05:02 -0700	[diff] [blame]	1558	#endif // XNN_NO_QS8_OPERATORS
				1559
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	1560	/************************** QU8 micro-kernels **************************/
				1561	#ifndef XNN_NO_QU8_OPERATORS
				1562	init_flags \|= XNN_INIT_FLAG_QU8;
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	1563
Frank Barchard	a962f1e	2021-08-02 13:52:15 -0700	[diff] [blame]	1564	#if XNN_ENABLE_ASSEMBLY
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	1565	if (cpuinfo_has_arm_neon_dot()) {
Frank Barchard	8b69802	2021-08-26 11:17:32 -0700	[diff] [blame]	1566	switch (cpuinfo_get_core(0)->uarch) {
				1567	case cpuinfo_uarch_cortex_a55:
Frank Barchard	a49e41f	2021-08-31 20:30:24 -0700	[diff] [blame]	1568	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_cortex_a55);
				1569	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_cortex_a55);
				1570	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1571	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16c4__neondot);
Frank Barchard	8b69802	2021-08-26 11:17:32 -0700	[diff] [blame]	1572	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				1573	xnn_params.qu8.gemm.mr = 4;
Frank Barchard	a49e41f	2021-08-31 20:30:24 -0700	[diff] [blame]	1574	xnn_params.qu8.gemm.nr = 16;
Frank Barchard	8b69802	2021-08-26 11:17:32 -0700	[diff] [blame]	1575	xnn_params.qu8.gemm.log2_kr = 2;
				1576	break;
				1577	default:
				1578	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_ld128);
				1579	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_ld128);
				1580	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1581	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1582	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				1583	xnn_params.qu8.gemm.mr = 4;
				1584	xnn_params.qu8.gemm.nr = 16;
				1585	xnn_params.qu8.gemm.log2_kr = 2;
				1586	break;
				1587	}
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	1588	} else {
				1589	switch (cpuinfo_get_core(0)->uarch) {
				1590	case cpuinfo_uarch_cortex_a53:
				1591	case cpuinfo_uarch_cortex_a55r0:
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	1592	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53);
				1593	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53);
				1594	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1595	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1596	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				1597	xnn_params.qu8.gemm.mr = 4;
				1598	xnn_params.qu8.gemm.nr = 16;
				1599	break;
Frank Barchard	f479a1c	2021-08-03 10:20:30 -0700	[diff] [blame]	1600
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	1601	case cpuinfo_uarch_cortex_a57:
				1602	case cpuinfo_uarch_cortex_a72:
				1603	case cpuinfo_uarch_cortex_a73:
				1604	case cpuinfo_uarch_cortex_a75:
				1605	case cpuinfo_uarch_cortex_a76:
				1606	case cpuinfo_uarch_exynos_m1:
				1607	case cpuinfo_uarch_exynos_m2:
				1608	case cpuinfo_uarch_exynos_m3:
				1609	case cpuinfo_uarch_exynos_m4:
				1610	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a75);
				1611	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a75);
				1612	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1613	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1614	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				1615	xnn_params.qu8.gemm.mr = 4;
				1616	xnn_params.qu8.gemm.nr = 16;
				1617	break;
Frank Barchard	f479a1c	2021-08-03 10:20:30 -0700	[diff] [blame]	1618
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	1619	case cpuinfo_uarch_kryo:
				1620	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16__neon_mlal_lane);
				1621	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16__neon_mlal_lane);
				1622	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1623	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1624	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				1625	xnn_params.qu8.gemm.mr = 4;
				1626	xnn_params.qu8.gemm.nr = 16;
				1627	break;
				1628
				1629	default:
				1630	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a75);
				1631	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a75);
				1632	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1633	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1634	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				1635	xnn_params.qu8.gemm.mr = 4;
				1636	xnn_params.qu8.gemm.nr = 16;
				1637	break;
				1638	}
Frank Barchard	f479a1c	2021-08-03 10:20:30 -0700	[diff] [blame]	1639	}
Frank Barchard	c37b8da	2021-09-01 00:35:19 -0700	[diff] [blame]	1640	#if XNN_MAX_UARCH_TYPES > 1
				1641	{
				1642	/* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
				1643	const uint32_t mr = xnn_params.qu8.gemm.mr;
				1644	const uint32_t nr = xnn_params.qu8.gemm.nr;
				1645	const uint32_t log2_kr = xnn_params.qu8.gemm.log2_kr;
				1646	for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
				1647	const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
				1648	if (uarch_info == NULL) {
				1649	/* No more microarchitectures in the system */
				1650	break;
				1651	}
				1652
				1653	switch (uarch_info->uarch) {
				1654	case cpuinfo_uarch_cortex_a53:
				1655	case cpuinfo_uarch_cortex_a55r0:
				1656	if (mr == 4 && nr == 16 && log2_kr == 0) {
				1657	xnn_params.qu8.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53;
				1658	xnn_params.qu8.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53;
				1659	}
				1660	break;
				1661
				1662	case cpuinfo_uarch_cortex_a55:
				1663	if (mr == 4 && nr == 16 && log2_kr == 2 && cpuinfo_has_arm_neon_dot()) {
				1664	xnn_params.qu8.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_cortex_a55;
				1665	xnn_params.qu8.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16c4__aarch64_neondot_cortex_a55;
				1666	}
				1667	break;
				1668	default:
				1669	break;
				1670	}
				1671	}
				1672	}
				1673	#endif // XNN_MAX_UARCH_TYPES > 1
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	1674	#else // !XNN_ENABLE_ASSEMBLY
				1675	if (cpuinfo_has_arm_neon_dot()) {
				1676	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16c4__neondot);
				1677	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16c4__neondot);
				1678	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1679	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16c4__neondot);
				1680	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				1681	xnn_params.qu8.gemm.mr = 4;
				1682	xnn_params.qu8.gemm.nr = 16;
				1683	xnn_params.qu8.gemm.log2_kr = 2;
				1684	} else {
				1685	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_4x16__neon_mlal_lane);
				1686	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_4x16__neon_mlal_lane);
				1687	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1688	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_rndnu_ukernel_1x16__neon_mlal_lane);
				1689	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
				1690	xnn_params.qu8.gemm.mr = 4;
				1691	xnn_params.qu8.gemm.nr = 16;
Marat Dukhan	947805b	2021-12-07 14:32:09 -0800	[diff] [blame]	1692	}
Frank Barchard	2025515	2021-08-11 14:01:45 -0700	[diff] [blame]	1693	#endif // XNN_ENABLE_ASSEMBLY
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1694
Frank Barchard	354cbc6	2021-09-27 21:42:41 -0700	[diff] [blame]	1695	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mul8;
Marat Dukhan	73a899a	2021-07-27 00:10:38 -0700	[diff] [blame]	1696	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
Frank Barchard	354cbc6	2021-09-27 21:42:41 -0700	[diff] [blame]	1697	xnn_params.qu8.dwconv[0].channel_tile = 16;
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	1698	xnn_params.qu8.dwconv[0].primary_tile = 9;
Frank Barchard	354cbc6	2021-09-27 21:42:41 -0700	[diff] [blame]	1699	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mul8;
Marat Dukhan	73a899a	2021-07-27 00:10:38 -0700	[diff] [blame]	1700	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_rndnu_neon_params;
Frank Barchard	354cbc6	2021-09-27 21:42:41 -0700	[diff] [blame]	1701	xnn_params.qu8.dwconv[1].channel_tile = 8;
Marat Dukhan	8172135	2021-07-15 18:26:08 -0700	[diff] [blame]	1702	xnn_params.qu8.dwconv[1].primary_tile = 25;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1703
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	1704	xnn_params.qu8.avgpool = (struct avgpool_parameters) {
				1705	.up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__neon_c8,
				1706	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__neon_c8,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1707	.mr = 9,
				1708	.qr = 8,
				1709	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	1710	xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
				1711	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__neon_c8,
				1712	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__neon_c8,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1713	.mr = 7,
				1714	};
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	1715	xnn_params.qu8.vadd = (struct vbinary_parameters) {
Frank Barchard	0a3093c	2021-08-31 09:58:11 -0700	[diff] [blame]	1716	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__neon_ld64_x32,
				1717	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__neon_ld64_x32,
				1718	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__neon_ld64_x32,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	1719	.init.qu8_addsub = xnn_init_qu8_add_minmax_neon_params,
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	1720	.element_tile = 8,
				1721	};
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	1722	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				1723	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__neonv8_ld64_x16,
				1724	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__neonv8_ld64_x16,
				1725	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__neonv8_ld64_x16,
				1726	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_neonv8_params,
				1727	.element_tile = 16,
				1728	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	1729	#endif // XNN_NO_QU8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1730
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	1731	/************************** S8 micro-kernels **************************/
				1732	#ifndef XNN_NO_S8_OPERATORS
				1733	init_flags \|= XNN_INIT_FLAG_S8;
				1734
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	1735	xnn_params.s8.clamp = (struct vunary_parameters) {
				1736	.ukernel = (xnn_univector_ukernel_function) xnn_s8_vclamp_ukernel__neon_x64,
				1737	.init.s8_minmax = xnn_init_s8_minmax_neon_params,
				1738	.element_tile = 64,
				1739	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	1740	xnn_params.s8.ibilinear = (struct ibilinear_parameters) {
				1741	.ukernel = (xnn_ibilinear_ukernel_function) xnn_s8_ibilinear_ukernel__neon_c16,
				1742	.pixel_tile = 1,
				1743	.channel_tile = 16,
				1744	};
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	1745	xnn_params.s8.maxpool = (struct maxpool_parameters) {
				1746	.ukernel = (xnn_maxpool_ukernel_function) xnn_s8_maxpool_minmax_ukernel_9p8x__neon_c16,
Marat Dukhan	dc5c148	2021-08-16 09:03:15 -0700	[diff] [blame]	1747	.init.s8 = xnn_init_s8_minmax_neon_params,
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	1748	.mr = 9,
				1749	.qr = 8,
				1750	};
				1751	#endif // XNN_NO_S8_OPERATORS
				1752
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1753	/************************** U8 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1754	#ifndef XNN_NO_U8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	1755	init_flags \|= XNN_INIT_FLAG_U8;
				1756
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	1757	xnn_params.u8.clamp = (struct vunary_parameters) {
				1758	.ukernel = (xnn_univector_ukernel_function) xnn_u8_vclamp_ukernel__neon_x64,
				1759	.init.u8_minmax = xnn_init_u8_minmax_neon_params,
				1760	.element_tile = 64,
				1761	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	1762	xnn_params.u8.ibilinear = (struct ibilinear_parameters) {
				1763	.ukernel = (xnn_ibilinear_ukernel_function) xnn_u8_ibilinear_ukernel__neon_c16,
				1764	.pixel_tile = 1,
				1765	.channel_tile = 16,
				1766	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1767	xnn_params.u8.maxpool = (struct maxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	1768	.ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__neon_c16,
Marat Dukhan	2ea50a0	2021-08-16 12:59:19 -0700	[diff] [blame]	1769	.init.u8 = xnn_init_u8_minmax_neon_params,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1770	.mr = 9,
				1771	.qr = 8,
				1772	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1773	xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
				1774	xnn_params.u8.rmax = xnn_u8_rmax_ukernel__neon;
				1775	#endif // XNN_NO_U8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1776
				1777	/************************** X8 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1778	#ifndef XNN_NO_X8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	1779	init_flags \|= XNN_INIT_FLAG_X8;
				1780
Marat Dukhan	98e054b	2021-09-13 09:43:50 -0700	[diff] [blame]	1781	xnn_params.x8.lut = xnn_x8_lut_ukernel__neon_tbx128x4_x64;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1782	xnn_params.x8.zip = (struct zip_parameters) {
				1783	.x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__neon,
				1784	.x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__neon,
				1785	.x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__neon,
				1786	.xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__neon,
				1787	};
				1788	#endif // XNN_NO_X8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1789
Frank Barchard	7e2cbb0	2020-06-12 01:22:13 -0700	[diff] [blame]	1790	/************************** F16 micro-kernels **************************/
				1791	#ifndef XNN_NO_F16_OPERATORS
Marat Dukhan	8d5d259	2020-06-19 12:48:57 -0700	[diff] [blame]	1792	if (cpuinfo_has_arm_neon_fp16_arith()) {
				1793	init_flags \|= XNN_INIT_FLAG_F16;
Frank Barchard	7c3826e	2021-06-07 15:14:16 -0700	[diff] [blame]	1794	xnn_params.f16.gemm.mr = 6;
				1795	xnn_params.f16.gemm.nr = 16;
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	1796
Frank Barchard	6b73c4f	2020-06-26 18:40:40 -0700	[diff] [blame]	1797	#if XNN_ENABLE_ASSEMBLY
Frank Barchard	7c3826e	2021-06-07 15:14:16 -0700	[diff] [blame]	1798	switch (cpuinfo_get_core(0)->uarch) {
				1799	case cpuinfo_uarch_cortex_a55:
				1800	xnn_params.f16.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__aarch64_neonfp16arith_cortex_a55);
				1801	break;
				1802
Frank Barchard	07f4a89	2021-06-07 18:26:08 -0700	[diff] [blame]	1803	case cpuinfo_uarch_cortex_a75:
Frank Barchard	7b48ddc	2021-06-11 13:00:49 -0700	[diff] [blame]	1804	case cpuinfo_uarch_cortex_x1:
Frank Barchard	07f4a89	2021-06-07 18:26:08 -0700	[diff] [blame]	1805	xnn_params.f16.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__aarch64_neonfp16arith_cortex_a75);
				1806	break;
				1807
Frank Barchard	7c3826e	2021-06-07 15:14:16 -0700	[diff] [blame]	1808	default:
				1809	xnn_params.f16.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__aarch64_neonfp16arith_ld32);
				1810	break;
				1811	}
Frank Barchard	6b73c4f	2020-06-26 18:40:40 -0700	[diff] [blame]	1812	xnn_params.f16.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_1x16__aarch64_neonfp16arith_ld32);
Frank Barchard	7c3826e	2021-06-07 15:14:16 -0700	[diff] [blame]	1813
				1814	#if XNN_MAX_UARCH_TYPES > 1
				1815	{
				1816	/* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
				1817	const uint32_t mr = xnn_params.f16.gemm.mr;
				1818	const uint32_t nr = xnn_params.f16.gemm.nr;
				1819	for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
				1820	const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
				1821	if (uarch_info == NULL) {
				1822	/* No more microarchitectures in the system */
				1823	break;
				1824	}
				1825
				1826	switch (uarch_info->uarch) {
				1827	case cpuinfo_uarch_cortex_a55:
				1828	if (mr == 6 && nr == 16) {
				1829	xnn_params.f16.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__aarch64_neonfp16arith_cortex_a55;
				1830	}
				1831	break;
Frank Barchard	07f4a89	2021-06-07 18:26:08 -0700	[diff] [blame]	1832
Frank Barchard	d2f454e	2021-06-08 10:47:16 -0700	[diff] [blame]	1833	case cpuinfo_uarch_cortex_a55r0:
				1834	if (mr == 6 && nr == 16) {
				1835	xnn_params.f16.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__neonfp16arith_ld64;
				1836	}
				1837	break;
				1838
Frank Barchard	07f4a89	2021-06-07 18:26:08 -0700	[diff] [blame]	1839	/* Cortex A75 is the medium core Exynos 9820 (M4) */
				1840	case cpuinfo_uarch_cortex_a75:
				1841	if (mr == 6 && nr == 16) {
				1842	xnn_params.f16.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__aarch64_neonfp16arith_cortex_a75;
				1843	}
				1844	break;
				1845
Frank Barchard	7c3826e	2021-06-07 15:14:16 -0700	[diff] [blame]	1846	default:
				1847	break;
				1848	}
				1849	}
				1850	}
				1851	#endif // XNN_MAX_UARCH_TYPES > 1
				1852	#else // XNN_ENABLE_ASSEMBLY
Frank Barchard	6b73c4f	2020-06-26 18:40:40 -0700	[diff] [blame]	1853	xnn_params.f16.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__neonfp16arith_ld64);
				1854	xnn_params.f16.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_1x16__neonfp16arith_ld64);
Frank Barchard	7c3826e	2021-06-07 15:14:16 -0700	[diff] [blame]	1855	#endif // XNN_ENABLE_ASSEMBLY
Frank Barchard	49b4dcc	2020-06-26 14:07:19 -0700	[diff] [blame]	1856	xnn_params.f16.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f16_igemm_minmax_ukernel_6x16__neonfp16arith_ld64);
Frank Barchard	49b4dcc	2020-06-26 14:07:19 -0700	[diff] [blame]	1857	xnn_params.f16.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f16_igemm_minmax_ukernel_1x16__neonfp16arith_ld64);
Marat Dukhan	c5a7a39	2021-05-21 16:04:31 -0700	[diff] [blame]	1858	xnn_params.f16.gemm.init.f16 = xnn_init_f16_scaleminmax_params;
Frank Barchard	49b4dcc	2020-06-26 14:07:19 -0700	[diff] [blame]	1859
				1860	xnn_params.f16.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f16_dwconv_minmax_ukernel_up16x4__neonfp16arith;
Marat Dukhan	c5a7a39	2021-05-21 16:04:31 -0700	[diff] [blame]	1861	xnn_params.f16.dwconv[0].init.f16 = xnn_init_f16_minmax_params;
Frank Barchard	49b4dcc	2020-06-26 14:07:19 -0700	[diff] [blame]	1862	xnn_params.f16.dwconv[0].channel_tile = 16;
				1863	xnn_params.f16.dwconv[0].primary_tile = 4;
				1864
				1865	xnn_params.f16.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f16_dwconv_minmax_ukernel_up16x9__neonfp16arith;
Marat Dukhan	c5a7a39	2021-05-21 16:04:31 -0700	[diff] [blame]	1866	xnn_params.f16.dwconv[1].init.f16 = xnn_init_f16_minmax_params;
Frank Barchard	49b4dcc	2020-06-26 14:07:19 -0700	[diff] [blame]	1867	xnn_params.f16.dwconv[1].channel_tile = 16;
				1868	xnn_params.f16.dwconv[1].primary_tile = 9;
				1869
				1870	xnn_params.f16.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f16_dwconv_minmax_ukernel_up8x25__neonfp16arith_acc2;
Marat Dukhan	c5a7a39	2021-05-21 16:04:31 -0700	[diff] [blame]	1871	xnn_params.f16.dwconv[2].init.f16 = xnn_init_f16_minmax_params;
Frank Barchard	49b4dcc	2020-06-26 14:07:19 -0700	[diff] [blame]	1872	xnn_params.f16.dwconv[2].channel_tile = 8;
				1873	xnn_params.f16.dwconv[2].primary_tile = 25;
				1874
Marat Dukhan	8d5d259	2020-06-19 12:48:57 -0700	[diff] [blame]	1875	xnn_params.f16.gavgpool = (struct gavgpool_parameters) {
				1876	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f16_gavgpool_minmax_ukernel_7x__neonfp16arith_c8,
				1877	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f16_gavgpool_minmax_ukernel_7p7x__neonfp16arith_c8,
				1878	.mr = 7,
				1879	};
Frank Barchard	01898c0	2020-06-23 21:49:50 -0700	[diff] [blame]	1880	xnn_params.f16.vadd = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	1881	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vadd_minmax_ukernel__neonfp16arith_x16,
				1882	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vaddc_minmax_ukernel__neonfp16arith_x16,
				1883	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vaddc_minmax_ukernel__neonfp16arith_x16,
Frank Barchard	01898c0	2020-06-23 21:49:50 -0700	[diff] [blame]	1884	.element_tile = 16,
				1885	};
Frank Barchard	0ea6a77	2020-09-09 15:26:31 -0700	[diff] [blame]	1886	xnn_params.f16.vmul = (struct vbinary_parameters) {
				1887	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vmul_minmax_ukernel__neonfp16arith_x16,
				1888	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vmulc_minmax_ukernel__neonfp16arith_x16,
				1889	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vmulc_minmax_ukernel__neonfp16arith_x16,
				1890	.element_tile = 16,
				1891	};
Frank Barchard	49b4dcc	2020-06-26 14:07:19 -0700	[diff] [blame]	1892	xnn_params.f16.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan	9531e9f	2020-07-24 15:25:02 -0700	[diff] [blame]	1893	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f16_vmulcaddc_minmax_ukernel_c8__neonfp16arith_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	1894	.init.f16 = xnn_init_f16_minmax_params,
Frank Barchard	49b4dcc	2020-06-26 14:07:19 -0700	[diff] [blame]	1895	.channel_tile = 8,
				1896	.row_tile = 2,
				1897	};
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	1898	xnn_params.f16.hswish = (xnn_univector_ukernel_function) xnn_f16_vhswish_ukernel__neonfp16arith_x16;
Marat Dukhan	8d5d259	2020-06-19 12:48:57 -0700	[diff] [blame]	1899	}
Frank Barchard	7e2cbb0	2020-06-12 01:22:13 -0700	[diff] [blame]	1900	#endif // XNN_NO_F16_OPERATORS
				1901
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1902	/************************** F32 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	1903	#ifndef XNN_NO_F32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	1904	init_flags \|= XNN_INIT_FLAG_F32;
				1905
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	1906	#if XNN_PLATFORM_IOS \|\| XNN_PLATFORM_MAC
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1907	#if XNN_ENABLE_ASSEMBLY
Frank Barchard	143a110	2021-06-15 09:15:34 -0700	[diff] [blame]	1908	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_prfm_cortex_a75);
				1909	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_prfm_cortex_a75);
				1910	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_prfm_cortex_a75);
				1911	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_prfm_cortex_a75);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1912	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1913	xnn_params.f32.gemm.mr = 6;
				1914	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1915	#else // !XNN_ENABLE_ASSEMBLY
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1916	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__neonfma_lane_ld64);
				1917	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__neonfma_lane_ld64);
				1918	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neonfma_lane_ld64);
				1919	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neonfma_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1920	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1921	xnn_params.f32.gemm.mr = 6;
				1922	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1923	#endif // XNN_ENABLE_ASSEMBLY
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	1924	#else // !XNN_PLATFORM_IOS && !XNN_PLATFORM_MAC
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1925	#if XNN_ENABLE_ASSEMBLY
				1926	switch (cpuinfo_get_core(0)->uarch) {
				1927	case cpuinfo_uarch_cortex_a57:
Frank Barchard	143a110	2021-06-15 09:15:34 -0700	[diff] [blame]	1928	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a75);
				1929	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a75);
				1930	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
				1931	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1932	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1933	xnn_params.f32.gemm.mr = 6;
				1934	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1935	break;
				1936	case cpuinfo_uarch_cortex_a72:
Frank Barchard	143a110	2021-06-15 09:15:34 -0700	[diff] [blame]	1937	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_prfm_cortex_a75);
				1938	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_prfm_cortex_a75);
				1939	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_prfm_cortex_a75);
				1940	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_prfm_cortex_a75);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1941	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1942	xnn_params.f32.gemm.mr = 4;
				1943	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1944	break;
				1945	case cpuinfo_uarch_cortex_a75:
				1946	case cpuinfo_uarch_cortex_a76:
				1947	case cpuinfo_uarch_exynos_m3:
				1948	case cpuinfo_uarch_exynos_m4:
Frank Barchard	143a110	2021-06-15 09:15:34 -0700	[diff] [blame]	1949	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_prfm_cortex_a75);
				1950	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_prfm_cortex_a75);
				1951	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_prfm_cortex_a75);
				1952	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_prfm_cortex_a75);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1953	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1954	xnn_params.f32.gemm.mr = 6;
				1955	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1956	break;
				1957	case cpuinfo_uarch_exynos_m1:
				1958	case cpuinfo_uarch_exynos_m2:
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1959	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8s4__neonfma);
				1960	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8s4__neonfma);
				1961	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8s4__neonfma);
				1962	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8s4__neonfma);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1963	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1964	xnn_params.f32.gemm.mr = 6;
				1965	xnn_params.f32.gemm.nr = 8;
				1966	xnn_params.f32.gemm.log2_sr = 2;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1967	break;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1968	case cpuinfo_uarch_cortex_a53:
				1969	case cpuinfo_uarch_cortex_a55r0:
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1970	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53);
				1971	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53);
				1972	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53);
				1973	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1974	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1975	xnn_params.f32.gemm.mr = 6;
				1976	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1977	break;
Frank Barchard	f975ee0	2021-11-05 16:01:00 -0700	[diff] [blame]	1978	case cpuinfo_uarch_cortex_a35:
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1979	case cpuinfo_uarch_cortex_a55:
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1980	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55);
				1981	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55);
				1982	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53);
				1983	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1984	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1985	xnn_params.f32.gemm.mr = 6;
				1986	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1987	break;
				1988	case cpuinfo_uarch_cortex_a73:
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1989	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a73);
				1990	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a73);
Frank Barchard	143a110	2021-06-15 09:15:34 -0700	[diff] [blame]	1991	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_prfm_cortex_a75);
				1992	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_prfm_cortex_a75);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	1993	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	1994	xnn_params.f32.gemm.mr = 6;
				1995	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1996	break;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	1997	case cpuinfo_uarch_cortex_a77:
				1998	case cpuinfo_uarch_exynos_m5:
				1999	case cpuinfo_uarch_kryo:
Frank Barchard	143a110	2021-06-15 09:15:34 -0700	[diff] [blame]	2000	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a75);
				2001	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a75);
				2002	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
				2003	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	2004	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2005	xnn_params.f32.gemm.mr = 4;
				2006	xnn_params.f32.gemm.nr = 8;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	2007	break;
Frank Barchard	990b2af	2021-06-14 11:49:15 -0700	[diff] [blame]	2008	case cpuinfo_uarch_cortex_a78:
				2009	case cpuinfo_uarch_cortex_x1:
				2010	default:
				2011	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_ld128);
Frank Barchard	79cd5f9	2021-06-21 17:34:59 -0700	[diff] [blame]	2012	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_ld128);
Frank Barchard	990b2af	2021-06-14 11:49:15 -0700	[diff] [blame]	2013	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neonfma_lane_ld64);
				2014	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neonfma_lane_ld64);
				2015	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
				2016	xnn_params.f32.gemm.mr = 6;
				2017	xnn_params.f32.gemm.nr = 8;
				2018	break;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	2019	}
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	2020	#if XNN_MAX_UARCH_TYPES > 1
				2021	{
				2022	/* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
				2023	const uint32_t mr = xnn_params.f32.gemm.mr;
				2024	const uint32_t nr = xnn_params.f32.gemm.nr;
				2025	const uint32_t log2_sr = xnn_params.f32.gemm.log2_sr;
				2026	for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
				2027	const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
				2028	if (uarch_info == NULL) {
				2029	/* No more microarchitectures in the system */
				2030	break;
				2031	}
				2032
				2033	switch (uarch_info->uarch) {
				2034	case cpuinfo_uarch_cortex_a53:
				2035	case cpuinfo_uarch_cortex_a55r0:
				2036	if (mr == 6 && nr == 8 && log2_sr == 0) {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2037	xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53;
				2038	xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53;
				2039	xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
				2040	xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	2041	} else if (mr == 4 && nr == 8 && log2_sr == 0) {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2042	xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a53;
				2043	xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a53;
				2044	xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
				2045	xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	2046	}
				2047	break;
				2048	case cpuinfo_uarch_cortex_a55:
				2049	if (mr == 6 && nr == 8 && log2_sr == 0) {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2050	xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55;
				2051	xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55;
				2052	xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
				2053	xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	2054	} else if (mr == 4 && nr == 8 && log2_sr == 0) {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2055	xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a55;
				2056	xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a55;
				2057	xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
				2058	xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
Marat Dukhan	05702cf	2020-03-26 15:41:33 -0700	[diff] [blame]	2059	}
				2060	break;
				2061	default:
				2062	break;
				2063	}
				2064	}
				2065	}
				2066	#endif // XNN_MAX_UARCH_TYPES > 1
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	2067	#else // !XNN_ENABLE_ASSEMBLY
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2068	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__neonfma_lane_ld64);
				2069	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__neonfma_lane_ld64);
				2070	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neonfma_lane_ld64);
				2071	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neonfma_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	2072	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2073	xnn_params.f32.gemm.mr = 6;
				2074	xnn_params.f32.gemm.nr = 8;
Marat Dukhan	31677ad	2020-10-13 23:59:31 -0700	[diff] [blame]	2075	#endif // XNN_ENABLE_ASSEMBLY
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	2076	#endif // XNN_PLATFORM_IOS \|\| XNN_PLATFORM_MAC
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2077	xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__neonfma_lane_ld64);
				2078	xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__neonfma_lane_ld64);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	2079	xnn_params.f32.gemm2.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2080	xnn_params.f32.gemm2.mr = 4;
				2081	xnn_params.f32.gemm2.nr = 2;
				2082
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	2083	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x3__neonfma;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	2084	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	f5425ea	2020-04-24 01:46:00 -0700	[diff] [blame]	2085	xnn_params.f32.dwconv[0].channel_tile = 8;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	2086	xnn_params.f32.dwconv[0].primary_tile = 3;
				2087
				2088	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x4__neonfma;
				2089	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_scalar_params;
				2090	xnn_params.f32.dwconv[1].channel_tile = 8;
				2091	xnn_params.f32.dwconv[1].primary_tile = 4;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2092
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	2093	#if XNN_PLATFORM_IOS \|\| XNN_PLATFORM_MAC
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	2094	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__neonfma;
				2095	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
				2096	xnn_params.f32.dwconv[2].channel_tile = 8;
				2097	xnn_params.f32.dwconv[2].primary_tile = 9;
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	2098	#else // !XNN_PLATFORM_IOS && !XNN_PLATFORM_MAC
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	2099	switch (cpuinfo_get_core(0)->uarch) {
				2100	case cpuinfo_uarch_kryo:
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	2101	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__neonfma;
				2102	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
				2103	xnn_params.f32.dwconv[2].channel_tile = 8;
				2104	xnn_params.f32.dwconv[2].primary_tile = 9;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	2105	break;
				2106	#if XNN_ENABLE_ASSEMBLY
				2107	case cpuinfo_uarch_cortex_a53:
				2108	case cpuinfo_uarch_cortex_a55r0:
				2109	case cpuinfo_uarch_cortex_a55:
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	2110	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x9__aarch64_neonfma_cortex_a55;
				2111	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
				2112	xnn_params.f32.dwconv[2].channel_tile = 4;
				2113	xnn_params.f32.dwconv[2].primary_tile = 9;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	2114	break;
				2115	#endif // XNN_ENABLE_ASSEMBLY
				2116	default:
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	2117	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__neonfma;
				2118	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
				2119	xnn_params.f32.dwconv[2].channel_tile = 8;
				2120	xnn_params.f32.dwconv[2].primary_tile = 9;
Frank Barchard	0d1052c	2020-03-23 17:28:13 -0700	[diff] [blame]	2121	break;
				2122	}
Marat Dukhan	dfe47b9	2020-12-14 02:48:43 -0800	[diff] [blame]	2123	#endif // XNN_PLATFORM_IOS && XNN_PLATFORM_MAC
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2124
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	2125	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x25__neonfma_acc2;
				2126	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_scalar_params;
				2127	xnn_params.f32.dwconv[3].channel_tile = 8;
				2128	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2129
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2130	xnn_params.f32.avgpool = (struct avgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	2131	.up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__neon_c4,
				2132	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__neon_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2133	.mr = 9,
				2134	.qr = 8,
				2135	};
				2136	xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	2137	.up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__neon_c4,
				2138	.mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__neon_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2139	.mr = 9,
				2140	.qr = 8,
				2141	};
				2142	xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	2143	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__neon_c4,
				2144	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__neon_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2145	.mr = 7,
				2146	};
				2147	xnn_params.f32.maxpool = (struct maxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	2148	.ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__neon_c4,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	2149	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2150	.mr = 9,
				2151	.qr = 8,
				2152	};
				2153	xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhan	ef25c6d	2020-07-24 00:59:40 -0700	[diff] [blame]	2154	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__neon_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2155	.mr = 4,
				2156	};
				2157	xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhan	ef25c6d	2020-07-24 00:59:40 -0700	[diff] [blame]	2158	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__neon_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2159	.mr = 9,
				2160	};
				2161	xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhan	ef25c6d	2020-07-24 00:59:40 -0700	[diff] [blame]	2162	.mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__neon_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2163	.mr = 9,
				2164	.qr = 8,
				2165	};
Marat Dukhan	660fd19	2020-03-10 04:55:30 -0700	[diff] [blame]	2166	xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
				2167	.ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__neonfma_c8,
Marat Dukhan	6972249	2019-11-11 19:55:50 -0800	[diff] [blame]	2168	.pixel_tile = 1,
				2169	.channel_tile = 8,
				2170	};
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	2171	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__neon_x8;
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	2172	xnn_params.f32.clamp = (struct vunary_parameters) {
				2173	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__neon_x8,
				2174	.init.f32_minmax = xnn_init_f32_minmax_scalar_params,
				2175	.element_tile = 8,
				2176	};
Marat Dukhan	b6bd4bc	2020-12-01 17:01:40 -0800	[diff] [blame]	2177	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__neonfma_rr1_lut16_p3_x16;
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	2178	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__neon_x16;
Marat Dukhan	2881333	2020-06-10 18:05:38 -0700	[diff] [blame]	2179	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__neon_x8;
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	2180	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__neon_x8;
Marat Dukhan	64e5251	2020-06-09 13:41:16 -0700	[diff] [blame]	2181	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__neonv8_x8;
				2182	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__neonv8_x8;
				2183	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__neonv8_x8;
				2184	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__neonv8_x8;
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	2185	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__neonfma_rr1_lut64_p2_nr2recps_x16;
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	2186	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__neon_x8;
Marat Dukhan	6804bbd	2020-06-30 19:26:11 -0700	[diff] [blame]	2187	xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__neon_sqrt_x4;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2188	xnn_params.f32.prelu = (struct prelu_parameters) {
Marat Dukhan	69c3f2c	2019-11-06 12:30:01 -0800	[diff] [blame]	2189	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__neon_2x8,
				2190	.row_tile = 2,
				2191	.channel_tile = 8,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2192	};
Marat Dukhan	1edc454	2020-01-27 12:40:13 -0800	[diff] [blame]	2193	xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__neonfma_lut64_p2_x16;
				2194	xnn_params.f32.rmax = xnn_f32_rmax_ukernel__neon;
Marat Dukhan	b1a0fc3	2019-12-02 19:32:02 -0800	[diff] [blame]	2195	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	2196	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__neon_x8,
				2197	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__neon_x8,
				2198	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__neon_x8,
Marat Dukhan	b1a0fc3	2019-12-02 19:32:02 -0800	[diff] [blame]	2199	.element_tile = 8,
				2200	};
Marat Dukhan	6918050	2019-12-06 15:00:31 -0800	[diff] [blame]	2201	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	2202	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__neon_x8,
				2203	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__neon_x8,
				2204	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__neon_x8,
Marat Dukhan	6918050	2019-12-06 15:00:31 -0800	[diff] [blame]	2205	.element_tile = 8,
				2206	};
Marat Dukhan	79e7f84	2019-12-05 14:35:50 -0800	[diff] [blame]	2207	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	2208	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__neon_x8,
				2209	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__neon_x8,
				2210	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__neon_x8,
Marat Dukhan	79e7f84	2019-12-05 14:35:50 -0800	[diff] [blame]	2211	.element_tile = 8,
				2212	};
				2213	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	2214	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__neon_x8,
				2215	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__neon_x8,
				2216	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__neon_x8,
Marat Dukhan	79e7f84	2019-12-05 14:35:50 -0800	[diff] [blame]	2217	.element_tile = 8,
				2218	};
Marat Dukhan	1e782c4	2019-11-21 17:02:40 -0800	[diff] [blame]	2219	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	2220	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__neon_x8,
				2221	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__neon_x8,
				2222	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__neon_x8,
Marat Dukhan	ca2733c	2019-11-15 23:21:17 -0800	[diff] [blame]	2223	.element_tile = 8,
				2224	};
Marat Dukhan	05f3f6d	2019-12-03 15:13:53 -0800	[diff] [blame]	2225	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	2226	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__neon_x8,
				2227	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__neon_x8,
				2228	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__neon_x8,
Marat Dukhan	05f3f6d	2019-12-03 15:13:53 -0800	[diff] [blame]	2229	.element_tile = 8,
				2230	};
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	2231	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	2232	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__neon_x8,
				2233	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__neon_x8,
				2234	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__neon_x8,
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	2235	.element_tile = 8,
				2236	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2237	xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan	9531e9f	2020-07-24 15:25:02 -0700	[diff] [blame]	2238	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__neonfma_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	2239	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	49e6ee9	2019-11-06 15:55:29 -0800	[diff] [blame]	2240	.channel_tile = 4,
				2241	.row_tile = 2,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2242	};
Marat Dukhan	efc47b8	2019-11-18 09:25:38 -0800	[diff] [blame]	2243	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	cfbed0a	2020-12-08 10:01:51 -0800	[diff] [blame]	2244	init_flags \|= XNN_INIT_FLAG_CHW_OPT;
				2245
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2246	xnn_params.f32.spmm = (struct spmm_parameters) {
Marat Dukhan	4baa2ac	2021-02-11 10:04:09 -0800	[diff] [blame]	2247	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_32x1__neonfma_pipelined,
				2248	.mr = 32,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2249	.nr = 1,
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	2250	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2251	xnn_params.f32.spmm2 = (struct spmm_parameters) {
Marat Dukhan	4baa2ac	2021-02-11 10:04:09 -0800	[diff] [blame]	2252	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_32x2__neonfma,
				2253	.mr = 32,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2254	.nr = 2,
				2255	};
				2256	xnn_params.f32.spmm4 = (struct spmm_parameters) {
Marat Dukhan	4baa2ac	2021-02-11 10:04:09 -0800	[diff] [blame]	2257	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_32x4__neonfma,
				2258	.mr = 32,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2259	.nr = 4,
				2260	};
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	2261	xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2262	.ukernel_with_symm_padding =
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	2263	(xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__neonfma_2x2,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2264	.output_channel_tile = 4,
				2265	.output_height_tile = 2,
				2266	.output_width_tile = 2,
				2267	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	2268	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
				2269	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__neonfma_3x4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2270	.output_width_tile = 4,
				2271	.output_height_tile = 3,
				2272	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	2273	xnn_params.f32.dwconv2d_chw_3x3s2 = (struct dwconv2d_chw_parameters) {
Marat Dukhan	82f0c32	2020-10-25 19:17:35 -0700	[diff] [blame]	2274	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__neonfma_2x4_acc2,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2275	.output_width_tile = 4,
Marat Dukhan	82f0c32	2020-10-25 19:17:35 -0700	[diff] [blame]	2276	.output_height_tile = 2,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2277	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	2278	xnn_params.f32.dwconv2d_chw_5x5 = (struct dwconv2d_chw_parameters) {
Marat Dukhan	149f0ea	2020-10-26 12:50:33 -0700	[diff] [blame]	2279	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5p2__neonfma_4x4,
Marat Dukhan	a99918a	2019-11-15 14:40:12 -0800	[diff] [blame]	2280	.output_width_tile = 4,
Marat Dukhan	149f0ea	2020-10-26 12:50:33 -0700	[diff] [blame]	2281	.output_height_tile = 4,
Marat Dukhan	a99918a	2019-11-15 14:40:12 -0800	[diff] [blame]	2282	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	2283	xnn_params.f32.dwconv2d_chw_5x5s2 = (struct dwconv2d_chw_parameters) {
				2284	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc2,
Marat Dukhan	a99918a	2019-11-15 14:40:12 -0800	[diff] [blame]	2285	.output_width_tile = 4,
				2286	.output_height_tile = 1,
				2287	};
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	2288	xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
				2289	.ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__neon_x4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2290	.channel_tile = 4,
				2291	};
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	2292	xnn_params.f32.ibilinear_chw = (struct ibilinear_chw_parameters) {
Artsiom Ablavatski	2202c81	2021-01-22 14:16:43 -0800	[diff] [blame]	2293	.ukernel = (xnn_ibilinear_chw_ukernel_function) xnn_f32_ibilinear_chw_ukernel__neonfma_p8,
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	2294	.channel_tile = 1,
Artsiom Ablavatski	b3ffd58	2021-03-31 13:00:08 -0700	[diff] [blame]	2295	.pixel_tile = 8,
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	2296	};
Marat Dukhan	efc47b8	2019-11-18 09:25:38 -0800	[diff] [blame]	2297	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2298	#endif // XNN_NO_F32_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	2299
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	2300	/************************* VCVT micro-kernels *************************/
				2301	#ifndef XNN_NO_VCVT_OPERATORS
				2302	init_flags \|= XNN_INIT_FLAG_VCVT;
				2303
				2304	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__neonfp16_x16;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	2305	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__neonfp16_x16;
Marat Dukhan	ed2d776	2021-12-03 23:51:19 -0800	[diff] [blame]	2306	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
				2307	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qs8_vcvt_ukernel__neonv8_x32,
				2308	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_neonv8_params,
				2309	.element_tile = 32,
				2310	};
				2311	xnn_params.vcvt.f32_to_qu8 = (struct vunary_parameters) {
				2312	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__neonv8_x32,
				2313	.init.f32_qu8_cvt = xnn_init_f32_qu8_cvt_neonv8_params,
				2314	.element_tile = 32,
				2315	};
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	2316	#endif // XNN_NO_VCVT_OPERATORS
				2317
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	2318	/************************** X32 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2319	#ifndef XNN_NO_X32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	2320	init_flags \|= XNN_INIT_FLAG_X32;
				2321
Marat Dukhan	57dccd8	2020-04-14 00:53:10 -0700	[diff] [blame]	2322	xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__neon;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2323	xnn_params.x32.zip = (struct zip_parameters) {
				2324	.x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__neon,
				2325	.x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__neon,
				2326	.x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__neon,
				2327	.xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__neon,
				2328	};
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	2329	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	ad71b9a	2020-11-20 00:01:51 -0800	[diff] [blame]	2330	xnn_params.x32.depthtospace2d_chw2hwc = (struct depthtospace2d_chw2hwc_parameters) {
				2331	.ukernel = (xnn_depthtospace2d_chw2hwc_ukernel_function) xnn_x32_depthtospace2d_chw2hwc_ukernel__scalar,
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	2332	.channel_tile = 1,
				2333	.pixel_tile = 1,
				2334	};
				2335	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2336	#endif // XNN_NO_X32_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	2337
Marat Dukhan	048931b	2020-11-24 20:53:54 -0800	[diff] [blame]	2338	/************************** XX micro-kernels **************************/
				2339	#ifndef XNN_NO_XX_OPERATORS
				2340	init_flags \|= XNN_INIT_FLAG_XX;
				2341
				2342	xnn_params.xx.copy = (xnn_univector_ukernel_function) xnn_xx_copy_ukernel__memcpy;
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	2343	xnn_params.xx.fill = (struct fill_parameters) {
				2344	.ukernel = (xnn_fill_ukernel_function) xnn_xx_fill_ukernel__neon_x64,
				2345	.row_tile = 1,
				2346	};
Marat Dukhan	0461f2d	2021-08-08 12:36:29 -0700	[diff] [blame]	2347	xnn_params.xx.pad = (struct pad_parameters) {
				2348	.ukernel = (xnn_pad_ukernel_function) xnn_xx_pad_ukernel__neon,
				2349	.row_tile = 1,
				2350	};
Marat Dukhan	048931b	2020-11-24 20:53:54 -0800	[diff] [blame]	2351	#endif
				2352
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	2353	#elif XNN_ARCH_X86 \|\| XNN_ARCH_X86_64
				2354	if (!cpuinfo_has_x86_sse2()) {
				2355	xnn_log_error("XNNPACK initialization failed: SSE2 is not supported");
				2356	return;
				2357	}
				2358
Marat Dukhan	5e35386	2021-06-15 09:03:25 -0700	[diff] [blame]	2359	/************************** QC8 micro-kernels **************************/
				2360	#ifndef XNN_NO_QC8_OPERATORS
				2361	init_flags \|= XNN_INIT_FLAG_QC8;
				2362
				2363	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
				2364	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x16c8__avx512skx);
				2365	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x16c8__avx512skx);
				2366	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x16c8__avx512skx);
				2367	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x16c8__avx512skx);
				2368	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_avx512_params;
				2369	xnn_params.qc8.gemm.mr = 4;
				2370	xnn_params.qc8.gemm.nr = 16;
				2371	xnn_params.qc8.gemm.log2_kr = 3;
				2372	} else if (cpuinfo_has_x86_xop()) {
				2373	// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
				2374	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
				2375	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
				2376	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
				2377	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
				2378	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2379	xnn_params.qc8.gemm.mr = 2;
				2380	xnn_params.qc8.gemm.nr = 4;
				2381	xnn_params.qc8.gemm.log2_kr = 3;
				2382	} else if (cpuinfo_has_x86_avx2()) {
				2383	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_3x8c8__avx2);
				2384	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_3x8c8__avx2);
				2385	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x8c8__avx2);
				2386	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x8c8__avx2);
				2387	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_avx2_params;
				2388	xnn_params.qc8.gemm.mr = 3;
				2389	xnn_params.qc8.gemm.nr = 8;
				2390	xnn_params.qc8.gemm.log2_kr = 3;
				2391	} else if (cpuinfo_has_x86_avx()) {
				2392	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
				2393	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
				2394	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
				2395	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
				2396	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2397	xnn_params.qc8.gemm.mr = 2;
				2398	xnn_params.qc8.gemm.nr = 4;
				2399	xnn_params.qc8.gemm.log2_kr = 3;
				2400	} else if (cpuinfo_has_x86_sse4_1()) {
				2401	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
				2402	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
				2403	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
				2404	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
				2405	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2406	xnn_params.qc8.gemm.mr = 3;
				2407	xnn_params.qc8.gemm.nr = 4;
				2408	xnn_params.qc8.gemm.log2_kr = 3;
				2409	} else {
				2410	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
				2411	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
				2412	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
				2413	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
				2414	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_sse2_params;
				2415	xnn_params.qc8.gemm.mr = 3;
				2416	xnn_params.qc8.gemm.nr = 4;
				2417	xnn_params.qc8.gemm.log2_kr = 3;
				2418	}
				2419
				2420	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
				2421	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up32x9__avx512skx_mul32;
				2422	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_avx512_params;
				2423	xnn_params.qc8.dwconv[0].channel_tile = 32;
				2424	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up32x25__avx512skx_mul32;
				2425	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_avx512_params;
				2426	xnn_params.qc8.dwconv[1].channel_tile = 32;
				2427	} else if (cpuinfo_has_x86_xop()) {
				2428	// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
Marat Dukhan	2848059	2021-07-27 23:52:27 -0700	[diff] [blame]	2429	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__xop_mul16_add16;
Marat Dukhan	5e35386	2021-06-15 09:03:25 -0700	[diff] [blame]	2430	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2431	xnn_params.qc8.dwconv[0].channel_tile = 16;
Marat Dukhan	2848059	2021-07-27 23:52:27 -0700	[diff] [blame]	2432	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__xop_mul16_add16;
Marat Dukhan	5e35386	2021-06-15 09:03:25 -0700	[diff] [blame]	2433	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2434	xnn_params.qc8.dwconv[1].channel_tile = 16;
				2435	} else if (cpuinfo_has_x86_avx2()) {
				2436	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__avx2_mul32;
				2437	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_avx2_params;
				2438	xnn_params.qc8.dwconv[0].channel_tile = 16;
				2439	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__avx2_mul32;
				2440	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_avx2_params;
				2441	xnn_params.qc8.dwconv[1].channel_tile = 16;
				2442	} else if (cpuinfo_has_x86_avx()) {
Marat Dukhan	2848059	2021-07-27 23:52:27 -0700	[diff] [blame]	2443	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__avx_mul16_add16;
Marat Dukhan	5e35386	2021-06-15 09:03:25 -0700	[diff] [blame]	2444	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2445	xnn_params.qc8.dwconv[0].channel_tile = 16;
Marat Dukhan	2848059	2021-07-27 23:52:27 -0700	[diff] [blame]	2446	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__avx_mul16_add16;
Marat Dukhan	5e35386	2021-06-15 09:03:25 -0700	[diff] [blame]	2447	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2448	xnn_params.qc8.dwconv[1].channel_tile = 16;
				2449	} else if (cpuinfo_has_x86_sse4_1()) {
				2450	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__sse41_mul16;
				2451	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2452	xnn_params.qc8.dwconv[0].channel_tile = 8;
				2453	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__sse41_mul16;
				2454	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_sse4_params;
				2455	xnn_params.qc8.dwconv[1].channel_tile = 8;
				2456	} else if (cpuinfo_has_x86_sse2()) {
				2457	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__sse2_mul16;
				2458	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_sse2_params;
				2459	xnn_params.qc8.dwconv[0].channel_tile = 8;
				2460	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__sse2_mul16;
				2461	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_sse2_params;
				2462	xnn_params.qc8.dwconv[1].channel_tile = 8;
				2463	}
				2464	xnn_params.qc8.dwconv[0].primary_tile = 9;
				2465	xnn_params.qc8.dwconv[1].primary_tile = 25;
				2466	#endif // XNN_NO_QC8_OPERATORS
				2467
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2468	/************************** QS8 micro-kernels **************************/
				2469	#ifndef XNN_NO_QS8_OPERATORS
				2470	init_flags \|= XNN_INIT_FLAG_QS8;
				2471
Marat Dukhan	bb00b1d	2020-08-10 11:37:23 -0700	[diff] [blame]	2472	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
Marat Dukhan	71855ee	2021-05-25 19:05:06 -0700	[diff] [blame]	2473	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_4x16c8__avx512skx);
				2474	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_4x16c8__avx512skx);
				2475	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x16c8__avx512skx);
				2476	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x16c8__avx512skx);
				2477	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx512_params;
Marat Dukhan	bb00b1d	2020-08-10 11:37:23 -0700	[diff] [blame]	2478	xnn_params.qs8.gemm.mr = 4;
				2479	xnn_params.qs8.gemm.nr = 16;
				2480	xnn_params.qs8.gemm.log2_kr = 3;
				2481	} else if (cpuinfo_has_x86_xop()) {
Marat Dukhan	75215d8	2020-08-07 23:08:03 -0700	[diff] [blame]	2482	// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
Marat Dukhan	c46e671	2021-06-01 19:00:16 -0700	[diff] [blame]	2483	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
				2484	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
				2485	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
				2486	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
				2487	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	75215d8	2020-08-07 23:08:03 -0700	[diff] [blame]	2488	xnn_params.qs8.gemm.mr = 2;
				2489	xnn_params.qs8.gemm.nr = 4;
				2490	xnn_params.qs8.gemm.log2_kr = 3;
				2491	} else if (cpuinfo_has_x86_avx2()) {
Marat Dukhan	9b474cf	2021-05-25 16:37:48 -0700	[diff] [blame]	2492	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x8c8__avx2);
				2493	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x8c8__avx2);
				2494	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x8c8__avx2);
				2495	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x8c8__avx2);
				2496	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx2_params;
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2497	xnn_params.qs8.gemm.mr = 3;
				2498	xnn_params.qs8.gemm.nr = 8;
				2499	xnn_params.qs8.gemm.log2_kr = 3;
Marat Dukhan	a3c1633	2021-04-02 15:03:27 -0700	[diff] [blame]	2500	} else if (cpuinfo_has_x86_avx()) {
Marat Dukhan	c46e671	2021-06-01 19:00:16 -0700	[diff] [blame]	2501	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
				2502	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
				2503	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
				2504	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
				2505	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	a3c1633	2021-04-02 15:03:27 -0700	[diff] [blame]	2506	xnn_params.qs8.gemm.mr = 2;
				2507	xnn_params.qs8.gemm.nr = 4;
				2508	xnn_params.qs8.gemm.log2_kr = 3;
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2509	} else if (cpuinfo_has_x86_sse4_1()) {
Marat Dukhan	c46e671	2021-06-01 19:00:16 -0700	[diff] [blame]	2510	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
				2511	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
				2512	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
				2513	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
				2514	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2515	xnn_params.qs8.gemm.mr = 3;
				2516	xnn_params.qs8.gemm.nr = 4;
				2517	xnn_params.qs8.gemm.log2_kr = 3;
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2518	} else {
Marat Dukhan	c46e671	2021-06-01 19:00:16 -0700	[diff] [blame]	2519	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
				2520	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
				2521	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
				2522	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
				2523	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse2_params;
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2524	xnn_params.qs8.gemm.mr = 3;
				2525	xnn_params.qs8.gemm.nr = 4;
				2526	xnn_params.qs8.gemm.log2_kr = 3;
				2527	}
				2528
Marat Dukhan	2ffc5e6	2020-09-06 22:33:38 -0700	[diff] [blame]	2529	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
Marat Dukhan	71855ee	2021-05-25 19:05:06 -0700	[diff] [blame]	2530	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up32x9__avx512skx_mul32;
				2531	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx512_params;
Marat Dukhan	2ffc5e6	2020-09-06 22:33:38 -0700	[diff] [blame]	2532	xnn_params.qs8.dwconv[0].channel_tile = 32;
Marat Dukhan	71855ee	2021-05-25 19:05:06 -0700	[diff] [blame]	2533	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up32x25__avx512skx_mul32;
				2534	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx512_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	2535	xnn_params.qs8.dwconv[1].channel_tile = 32;
Marat Dukhan	3fd4e27	2021-04-10 11:16:42 -0700	[diff] [blame]	2536	} else if (cpuinfo_has_x86_xop()) {
				2537	// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
Marat Dukhan	02f06e3	2021-07-27 14:33:47 -0700	[diff] [blame]	2538	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__xop_mul16_add16;
Marat Dukhan	caf4831	2021-06-01 20:20:58 -0700	[diff] [blame]	2539	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	3fd4e27	2021-04-10 11:16:42 -0700	[diff] [blame]	2540	xnn_params.qs8.dwconv[0].channel_tile = 16;
Marat Dukhan	0966856	2021-07-26 16:52:20 -0700	[diff] [blame]	2541	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__xop_mul16_add16;
Marat Dukhan	caf4831	2021-06-01 20:20:58 -0700	[diff] [blame]	2542	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	2543	xnn_params.qs8.dwconv[1].channel_tile = 16;
Marat Dukhan	2ffc5e6	2020-09-06 22:33:38 -0700	[diff] [blame]	2544	} else if (cpuinfo_has_x86_avx2()) {
Marat Dukhan	9b474cf	2021-05-25 16:37:48 -0700	[diff] [blame]	2545	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__avx2_mul32;
				2546	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx2_params;
Marat Dukhan	d65a152	2020-08-04 19:28:18 -0700	[diff] [blame]	2547	xnn_params.qs8.dwconv[0].channel_tile = 16;
Marat Dukhan	9b474cf	2021-05-25 16:37:48 -0700	[diff] [blame]	2548	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__avx2_mul32;
				2549	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_avx2_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	2550	xnn_params.qs8.dwconv[1].channel_tile = 16;
Marat Dukhan	fa0ab85	2021-04-02 17:30:49 -0700	[diff] [blame]	2551	} else if (cpuinfo_has_x86_avx()) {
Marat Dukhan	0966856	2021-07-26 16:52:20 -0700	[diff] [blame]	2552	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__avx_mul16_add16;
Marat Dukhan	caf4831	2021-06-01 20:20:58 -0700	[diff] [blame]	2553	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	fa0ab85	2021-04-02 17:30:49 -0700	[diff] [blame]	2554	xnn_params.qs8.dwconv[0].channel_tile = 16;
Marat Dukhan	0966856	2021-07-26 16:52:20 -0700	[diff] [blame]	2555	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__avx_mul16_add16;
Marat Dukhan	caf4831	2021-06-01 20:20:58 -0700	[diff] [blame]	2556	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	2557	xnn_params.qs8.dwconv[1].channel_tile = 16;
Marat Dukhan	d65a152	2020-08-04 19:28:18 -0700	[diff] [blame]	2558	} else if (cpuinfo_has_x86_sse4_1()) {
Marat Dukhan	0966856	2021-07-26 16:52:20 -0700	[diff] [blame]	2559	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up8x9__sse41_mul16_add16;
Marat Dukhan	caf4831	2021-06-01 20:20:58 -0700	[diff] [blame]	2560	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	d65a152	2020-08-04 19:28:18 -0700	[diff] [blame]	2561	xnn_params.qs8.dwconv[0].channel_tile = 8;
Marat Dukhan	0966856	2021-07-26 16:52:20 -0700	[diff] [blame]	2562	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up8x25__sse41_mul16_add16;
Marat Dukhan	caf4831	2021-06-01 20:20:58 -0700	[diff] [blame]	2563	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	2564	xnn_params.qs8.dwconv[1].channel_tile = 8;
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2565	} else if (cpuinfo_has_x86_sse2()) {
Marat Dukhan	0966856	2021-07-26 16:52:20 -0700	[diff] [blame]	2566	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up8x9__sse2_mul16_add16;
Marat Dukhan	caf4831	2021-06-01 20:20:58 -0700	[diff] [blame]	2567	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse2_params;
Marat Dukhan	d65a152	2020-08-04 19:28:18 -0700	[diff] [blame]	2568	xnn_params.qs8.dwconv[0].channel_tile = 8;
Marat Dukhan	0966856	2021-07-26 16:52:20 -0700	[diff] [blame]	2569	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up8x25__sse2_mul16_add16;
Marat Dukhan	caf4831	2021-06-01 20:20:58 -0700	[diff] [blame]	2570	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse2_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	2571	xnn_params.qs8.dwconv[1].channel_tile = 8;
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2572	}
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2573	xnn_params.qs8.dwconv[0].primary_tile = 9;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	2574	xnn_params.qs8.dwconv[1].primary_tile = 25;
Marat Dukhan	9e0b539	2020-08-07 02:29:34 -0700	[diff] [blame]	2575
				2576	if (cpuinfo_has_x86_sse4_1()) {
				2577	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				2578	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__sse41_c8_acc2,
				2579	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__sse41_c8_acc2,
				2580	.mr = 7,
				2581	};
				2582	} else if (cpuinfo_has_x86_ssse3()) {
				2583	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				2584	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__ssse3_c8_acc2,
				2585	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__ssse3_c8_acc2,
				2586	.mr = 7,
				2587	};
				2588	} else if (cpuinfo_has_x86_sse2()) {
				2589	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				2590	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__sse2_c8_acc2,
				2591	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__sse2_c8_acc2,
				2592	.mr = 7,
				2593	};
				2594	}
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	2595
Marat Dukhan	e76049a	2021-07-22 14:48:59 -0700	[diff] [blame]	2596	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
				2597	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				2598	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__avx512skx_mul32_ld128_x16,
				2599	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__avx512skx_mul32_ld128_x16,
				2600	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__avx512skx_mul32_ld128_x16,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2601	.init.qs8_addsub = xnn_init_qs8_add_minmax_avx512_params,
Marat Dukhan	e76049a	2021-07-22 14:48:59 -0700	[diff] [blame]	2602	.element_tile = 16,
				2603	};
				2604	} else if (cpuinfo_has_x86_xop()) {
Marat Dukhan	bb9225e	2020-09-06 22:40:56 -0700	[diff] [blame]	2605	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				2606	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__xop_mul32_ld32_x8,
				2607	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__xop_mul32_ld32_x8,
				2608	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__xop_mul32_ld32_x8,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2609	.init.qs8_addsub = xnn_init_qs8_add_minmax_sse4_mul32_params,
Marat Dukhan	bb9225e	2020-09-06 22:40:56 -0700	[diff] [blame]	2610	.element_tile = 8,
				2611	};
Marat Dukhan	3eac69c	2021-07-21 01:42:29 -0700	[diff] [blame]	2612	} else if (cpuinfo_has_x86_avx2()) {
Marat Dukhan	7679b1e	2021-07-20 18:32:23 -0700	[diff] [blame]	2613	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				2614	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__avx2_mul32_ld64_x16,
				2615	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__avx2_mul32_ld64_x16,
				2616	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__avx2_mul32_ld64_x16,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2617	.init.qs8_addsub = xnn_init_qs8_add_minmax_avx2_params,
Marat Dukhan	7679b1e	2021-07-20 18:32:23 -0700	[diff] [blame]	2618	.element_tile = 16,
				2619	};
Marat Dukhan	e9c4b96	2021-04-02 16:56:55 -0700	[diff] [blame]	2620	} else if (cpuinfo_has_x86_avx()) {
				2621	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				2622	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__avx_mul32_ld32_x8,
				2623	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__avx_mul32_ld32_x8,
				2624	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__avx_mul32_ld32_x8,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2625	.init.qs8_addsub = xnn_init_qs8_add_minmax_sse4_mul32_params,
Marat Dukhan	e9c4b96	2021-04-02 16:56:55 -0700	[diff] [blame]	2626	.element_tile = 8,
				2627	};
Marat Dukhan	bb9225e	2020-09-06 22:40:56 -0700	[diff] [blame]	2628	} else if (cpuinfo_has_x86_sse4_1()) {
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	2629	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				2630	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__sse41_mul16_ld64_x8,
				2631	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__sse41_mul16_ld64_x8,
				2632	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__sse41_mul16_ld64_x8,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2633	.init.qs8_addsub = xnn_init_qs8_add_minmax_sse4_mul16_params,
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	2634	.element_tile = 8,
				2635	};
				2636	} else {
				2637	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				2638	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__sse2_mul16_ld64_x8,
				2639	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__sse2_mul16_ld64_x8,
				2640	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__sse2_mul16_ld64_x8,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2641	.init.qs8_addsub = xnn_init_qs8_add_minmax_sse2_params,
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	2642	.element_tile = 8,
				2643	};
				2644	}
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	2645	if (cpuinfo_has_x86_avx()) {
				2646	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				2647	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__avx_mul16_ld64_x16,
				2648	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__avx_mul16_ld64_x16,
				2649	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__avx_mul16_ld64_x16,
				2650	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_sse4_params,
				2651	.element_tile = 16,
				2652	};
				2653	} else if (cpuinfo_has_x86_sse4_1()) {
				2654	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				2655	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__sse41_mul16_ld64_x16,
				2656	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__sse41_mul16_ld64_x16,
				2657	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__sse41_mul16_ld64_x16,
				2658	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_sse4_params,
				2659	.element_tile = 16,
				2660	};
				2661	} else {
				2662	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				2663	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__sse2_mul16_ld64_x8,
				2664	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__sse2_mul16_ld64_x8,
				2665	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__sse2_mul16_ld64_x8,
				2666	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_sse2_params,
				2667	.element_tile = 8,
				2668	};
				2669	}
Marat Dukhan	07e5040	2020-08-05 17:16:53 -0700	[diff] [blame]	2670	#endif // XNN_NO_QS8_OPERATORS
Marat Dukhan	16f1e1a	2020-08-04 16:38:22 -0700	[diff] [blame]	2671
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	2672	/************************** QU8 micro-kernels **************************/
				2673	#ifndef XNN_NO_QU8_OPERATORS
				2674	init_flags \|= XNN_INIT_FLAG_QU8;
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	2675
Marat Dukhan	3cf2e22	2021-07-08 11:38:45 -0700	[diff] [blame]	2676	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
				2677	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_4x16c8__avx512skx);
				2678	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_4x16c8__avx512skx);
				2679	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x16c8__avx512skx);
				2680	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x16c8__avx512skx);
				2681	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_avx512_params;
				2682	xnn_params.qu8.gemm.mr = 4;
				2683	xnn_params.qu8.gemm.nr = 16;
				2684	xnn_params.qu8.gemm.log2_kr = 3;
				2685	} else if (cpuinfo_has_x86_xop()) {
Marat Dukhan	902ef7f	2021-07-02 16:11:06 -0700	[diff] [blame]	2686	// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
				2687	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
				2688	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
				2689	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
				2690	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
				2691	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2692	xnn_params.qu8.gemm.mr = 2;
				2693	xnn_params.qu8.gemm.nr = 4;
				2694	xnn_params.qu8.gemm.log2_kr = 3;
				2695	} else if (cpuinfo_has_x86_avx2()) {
				2696	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2);
				2697	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2);
				2698	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2);
				2699	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2);
				2700	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_avx2_params;
				2701	xnn_params.qu8.gemm.mr = 3;
				2702	xnn_params.qu8.gemm.nr = 8;
				2703	xnn_params.qu8.gemm.log2_kr = 3;
				2704	} else if (cpuinfo_has_x86_avx()) {
				2705	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
				2706	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
				2707	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
				2708	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
				2709	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2710	xnn_params.qu8.gemm.mr = 2;
				2711	xnn_params.qu8.gemm.nr = 4;
				2712	xnn_params.qu8.gemm.log2_kr = 3;
				2713	} else if (cpuinfo_has_x86_sse4_1()) {
				2714	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
				2715	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
				2716	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
				2717	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
				2718	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2719	xnn_params.qu8.gemm.mr = 3;
				2720	xnn_params.qu8.gemm.nr = 4;
				2721	xnn_params.qu8.gemm.log2_kr = 3;
				2722	} else {
				2723	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
				2724	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
				2725	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
				2726	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
				2727	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2728	xnn_params.qu8.gemm.mr = 3;
				2729	xnn_params.qu8.gemm.nr = 4;
				2730	xnn_params.qu8.gemm.log2_kr = 3;
				2731	}
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2732
Marat Dukhan	abee3a7	2021-07-09 09:04:52 -0700	[diff] [blame]	2733	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
				2734	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up32x9__avx512skx_mul32;
				2735	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_avx512_params;
				2736	xnn_params.qu8.dwconv[0].channel_tile = 32;
				2737	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up32x25__avx512skx_mul32;
				2738	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_avx512_params;
				2739	xnn_params.qu8.dwconv[1].channel_tile = 32;
				2740	} else if (cpuinfo_has_x86_xop()) {
				2741	// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
				2742	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up16x9__xop_mul32;
				2743	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2744	xnn_params.qu8.dwconv[0].channel_tile = 16;
				2745	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up16x25__xop_mul32;
				2746	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2747	xnn_params.qu8.dwconv[1].channel_tile = 16;
				2748	} else if (cpuinfo_has_x86_avx2()) {
				2749	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up16x9__avx2_mul32;
				2750	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_avx2_params;
				2751	xnn_params.qu8.dwconv[0].channel_tile = 16;
				2752	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up16x25__avx2_mul32;
				2753	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_avx2_params;
				2754	xnn_params.qu8.dwconv[1].channel_tile = 16;
				2755	} else if (cpuinfo_has_x86_avx()) {
Marat Dukhan	caa7fc7	2021-07-27 07:48:24 -0700	[diff] [blame]	2756	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up16x9__avx_mul16;
Marat Dukhan	abee3a7	2021-07-09 09:04:52 -0700	[diff] [blame]	2757	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2758	xnn_params.qu8.dwconv[0].channel_tile = 16;
Marat Dukhan	caa7fc7	2021-07-27 07:48:24 -0700	[diff] [blame]	2759	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up16x25__avx_mul16;
Marat Dukhan	abee3a7	2021-07-09 09:04:52 -0700	[diff] [blame]	2760	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2761	xnn_params.qu8.dwconv[1].channel_tile = 16;
				2762	} else if (cpuinfo_has_x86_sse4_1()) {
				2763	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up8x9__sse41_mul16;
				2764	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2765	xnn_params.qu8.dwconv[0].channel_tile = 8;
				2766	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up8x25__sse41_mul16;
				2767	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2768	xnn_params.qu8.dwconv[1].channel_tile = 8;
				2769	} else if (cpuinfo_has_x86_sse2()) {
				2770	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up8x9__sse2_mul16;
				2771	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2772	xnn_params.qu8.dwconv[0].channel_tile = 8;
				2773	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up8x25__sse2_mul16;
				2774	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
				2775	xnn_params.qu8.dwconv[1].channel_tile = 8;
				2776	}
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	2777	xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhan	abee3a7	2021-07-09 09:04:52 -0700	[diff] [blame]	2778	xnn_params.qu8.dwconv[1].primary_tile = 25;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2779
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	2780	xnn_params.qu8.avgpool = (struct avgpool_parameters) {
				2781	.up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__sse2_c8,
				2782	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__sse2_c8,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2783	.mr = 9,
				2784	.qr = 8,
				2785	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	2786	xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
				2787	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__sse2_c8,
				2788	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__sse2_c8,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2789	.mr = 7,
				2790	};
Marat Dukhan	3eac69c	2021-07-21 01:42:29 -0700	[diff] [blame]	2791
Marat Dukhan	e76049a	2021-07-22 14:48:59 -0700	[diff] [blame]	2792	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
				2793	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				2794	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__avx512skx_mul32_ld128_x16,
				2795	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__avx512skx_mul32_ld128_x16,
				2796	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__avx512skx_mul32_ld128_x16,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2797	.init.qu8_addsub = xnn_init_qu8_add_minmax_avx512_params,
Marat Dukhan	e76049a	2021-07-22 14:48:59 -0700	[diff] [blame]	2798	.element_tile = 16,
				2799	};
				2800	} else if (cpuinfo_has_x86_xop()) {
Marat Dukhan	3eac69c	2021-07-21 01:42:29 -0700	[diff] [blame]	2801	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				2802	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__xop_mul32_ld32_x8,
				2803	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__xop_mul32_ld32_x8,
				2804	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__xop_mul32_ld32_x8,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2805	.init.qu8_addsub = xnn_init_qu8_add_minmax_sse4_params,
Marat Dukhan	3eac69c	2021-07-21 01:42:29 -0700	[diff] [blame]	2806	.element_tile = 8,
				2807	};
				2808	} else if (cpuinfo_has_x86_avx2()) {
				2809	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				2810	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__avx2_mul32_ld64_x16,
				2811	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__avx2_mul32_ld64_x16,
				2812	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__avx2_mul32_ld64_x16,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2813	.init.qu8_addsub = xnn_init_qu8_add_minmax_avx2_params,
Marat Dukhan	3eac69c	2021-07-21 01:42:29 -0700	[diff] [blame]	2814	.element_tile = 16,
				2815	};
				2816	} else if (cpuinfo_has_x86_avx()) {
				2817	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				2818	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__avx_mul32_ld32_x8,
				2819	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__avx_mul32_ld32_x8,
				2820	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__avx_mul32_ld32_x8,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2821	.init.qu8_addsub = xnn_init_qu8_add_minmax_sse4_params,
Marat Dukhan	3eac69c	2021-07-21 01:42:29 -0700	[diff] [blame]	2822	.element_tile = 8,
				2823	};
				2824	} else if (cpuinfo_has_x86_sse4_1()) {
				2825	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				2826	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__sse41_mul16_ld64_x8,
				2827	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__sse41_mul16_ld64_x8,
				2828	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__sse41_mul16_ld64_x8,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2829	.init.qu8_addsub = xnn_init_qu8_add_minmax_sse2_params,
Marat Dukhan	3eac69c	2021-07-21 01:42:29 -0700	[diff] [blame]	2830	.element_tile = 8,
				2831	};
				2832	} else {
				2833	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				2834	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__sse2_mul16_ld64_x8,
				2835	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__sse2_mul16_ld64_x8,
				2836	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__sse2_mul16_ld64_x8,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	2837	.init.qu8_addsub = xnn_init_qu8_add_minmax_sse2_params,
Marat Dukhan	3eac69c	2021-07-21 01:42:29 -0700	[diff] [blame]	2838	.element_tile = 8,
				2839	};
				2840	}
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	2841	if (cpuinfo_has_x86_avx()) {
				2842	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				2843	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__avx_mul16_ld64_x16,
				2844	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__avx_mul16_ld64_x16,
				2845	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__avx_mul16_ld64_x16,
				2846	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_sse2_params,
				2847	.element_tile = 16,
				2848	};
				2849	} else if (cpuinfo_has_x86_sse4_1()) {
				2850	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				2851	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__sse41_mul16_ld64_x16,
				2852	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__sse41_mul16_ld64_x16,
				2853	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__sse41_mul16_ld64_x16,
				2854	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_sse2_params,
				2855	.element_tile = 16,
				2856	};
				2857	} else {
				2858	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				2859	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__sse2_mul16_ld64_x8,
				2860	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__sse2_mul16_ld64_x8,
				2861	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__sse2_mul16_ld64_x8,
				2862	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_sse2_params,
				2863	.element_tile = 8,
				2864	};
				2865	}
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	2866	#endif // XNN_NO_QU8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	2867
				2868	/************************** U8 micro-kernels **************************/
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	2869	#ifndef XNN_NO_S8_OPERATORS
				2870	init_flags \|= XNN_INIT_FLAG_S8;
				2871
				2872	if (cpuinfo_has_x86_sse4_1()) {
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	2873	xnn_params.s8.clamp = (struct vunary_parameters) {
				2874	.ukernel = (xnn_univector_ukernel_function) xnn_s8_vclamp_ukernel__sse41_x64,
				2875	.init.s8_minmax = xnn_init_s8_minmax_sse4_params,
				2876	.element_tile = 64,
				2877	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	2878	xnn_params.s8.ibilinear = (struct ibilinear_parameters) {
				2879	.ukernel = (xnn_ibilinear_ukernel_function) xnn_s8_ibilinear_ukernel__sse41_c16,
				2880	.pixel_tile = 1,
				2881	.channel_tile = 16,
				2882	};
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	2883	xnn_params.s8.maxpool = (struct maxpool_parameters) {
				2884	.ukernel = (xnn_maxpool_ukernel_function) xnn_s8_maxpool_minmax_ukernel_9p8x__sse41_c16,
				2885	.init.s8 = xnn_init_s8_minmax_sse4_params,
				2886	.mr = 9,
				2887	.qr = 8,
				2888	};
				2889	} else {
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	2890	xnn_params.s8.clamp = (struct vunary_parameters) {
				2891	.ukernel = (xnn_univector_ukernel_function) xnn_s8_vclamp_ukernel__sse2_x64,
				2892	.init.s8_minmax = xnn_init_s8_minmax_sse2_params,
				2893	.element_tile = 64,
				2894	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	2895	xnn_params.s8.ibilinear = (struct ibilinear_parameters) {
				2896	.ukernel = (xnn_ibilinear_ukernel_function) xnn_s8_ibilinear_ukernel__sse2_c8,
				2897	.pixel_tile = 1,
				2898	.channel_tile = 8,
				2899	};
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	2900	xnn_params.s8.maxpool = (struct maxpool_parameters) {
				2901	.ukernel = (xnn_maxpool_ukernel_function) xnn_s8_maxpool_minmax_ukernel_9p8x__sse2_c16,
				2902	.init.s8 = xnn_init_s8_minmax_sse2_params,
				2903	.mr = 9,
				2904	.qr = 8,
				2905	};
				2906	}
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	2907	#endif // XNN_NO_S8_OPERATORS
Marat Dukhan	2314753	2021-08-16 07:26:56 -0700	[diff] [blame]	2908
				2909	/************************** U8 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2910	#ifndef XNN_NO_U8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	2911	init_flags \|= XNN_INIT_FLAG_U8;
				2912
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	2913	xnn_params.u8.clamp = (struct vunary_parameters) {
				2914	.ukernel = (xnn_univector_ukernel_function) xnn_u8_vclamp_ukernel__sse2_x64,
				2915	.init.u8_minmax = xnn_init_u8_minmax_sse2_params,
				2916	.element_tile = 64,
				2917	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	2918	if (cpuinfo_has_x86_sse4_1()) {
				2919	xnn_params.u8.ibilinear = (struct ibilinear_parameters) {
				2920	.ukernel = (xnn_ibilinear_ukernel_function) xnn_u8_ibilinear_ukernel__sse41_c16,
				2921	.pixel_tile = 1,
				2922	.channel_tile = 16,
				2923	};
				2924	} else {
				2925	xnn_params.u8.ibilinear = (struct ibilinear_parameters) {
				2926	.ukernel = (xnn_ibilinear_ukernel_function) xnn_u8_ibilinear_ukernel__sse2_c8,
				2927	.pixel_tile = 1,
				2928	.channel_tile = 8,
				2929	};
				2930	}
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2931	xnn_params.u8.maxpool = (struct maxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	2932	.ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__sse2_c16,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	2933	.init.u8 = xnn_init_u8_minmax_sse2_params,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2934	.mr = 9,
				2935	.qr = 8,
				2936	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2937	xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
				2938	xnn_params.u8.rmax = xnn_u8_rmax_ukernel__sse2;
				2939	#endif // XNN_NO_U8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	2940
				2941	/************************** X8 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2942	#ifndef XNN_NO_X8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	2943	init_flags \|= XNN_INIT_FLAG_X8;
				2944
Marat Dukhan	98e054b	2021-09-13 09:43:50 -0700	[diff] [blame]	2945	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
				2946	xnn_params.x8.lut = xnn_x8_lut_ukernel__avx512skx_vpshufb_x64;
				2947	} else if (cpuinfo_has_x86_avx2()) {
				2948	xnn_params.x8.lut = xnn_x8_lut_ukernel__avx2_x128;
				2949	} else if (cpuinfo_has_x86_avx()) {
				2950	xnn_params.x8.lut = xnn_x8_lut_ukernel__avx_x64;
				2951	} else {
				2952	// Note: SSSE3 version is usually slower than scalar
				2953	xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar_x4;
				2954	}
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2955	xnn_params.x8.zip = (struct zip_parameters) {
				2956	.x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__sse2,
				2957	.x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__sse2,
				2958	.x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__sse2,
				2959	.xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__sse2,
				2960	};
				2961	#endif // XNN_NO_X8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	2962
				2963	/************************** F32 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	2964	#ifndef XNN_NO_F32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	2965	init_flags \|= XNN_INIT_FLAG_F32;
				2966
Marat Dukhan	0f349c4	2019-11-27 11:58:54 -0800	[diff] [blame]	2967	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2968	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_7x16__avx512f_broadcast);
				2969	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_7x16__avx512f_broadcast);
				2970	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x16__avx512f_broadcast);
				2971	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x16__avx512f_broadcast);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	2972	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2973	xnn_params.f32.gemm.mr = 7;
				2974	xnn_params.f32.gemm.nr = 16;
Marat Dukhan	0f349c4	2019-11-27 11:58:54 -0800	[diff] [blame]	2975	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_fma3()) {
Marat Dukhan	2712132	2019-12-09 14:57:40 -0800	[diff] [blame]	2976	switch (cpuinfo_get_core(0)->uarch) {
				2977	case cpuinfo_uarch_zen:
Marat Dukhan	b3801eb	2020-03-12 13:41:11 -0700	[diff] [blame]	2978	case cpuinfo_uarch_dhyana:
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2979	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x16s4__fma3_broadcast);
				2980	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x16s4__fma3_broadcast);
				2981	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x16s4__fma3_broadcast);
				2982	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x16s4__fma3_broadcast);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	2983	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_avx_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2984	xnn_params.f32.gemm.mr = 4;
				2985	xnn_params.f32.gemm.nr = 16;
				2986	xnn_params.f32.gemm.log2_sr = 2;
Marat Dukhan	2712132	2019-12-09 14:57:40 -0800	[diff] [blame]	2987	break;
				2988	default:
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2989	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_5x16__fma3_broadcast);
				2990	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_5x16__fma3_broadcast);
				2991	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x16__fma3_broadcast);
				2992	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x16__fma3_broadcast);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	2993	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_avx_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2994	xnn_params.f32.gemm.mr = 5;
				2995	xnn_params.f32.gemm.nr = 16;
Marat Dukhan	2712132	2019-12-09 14:57:40 -0800	[diff] [blame]	2996	break;
				2997	}
Marat Dukhan	1025ea3	2019-11-21 16:01:08 -0800	[diff] [blame]	2998	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	2999	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_5x16__avx_broadcast);
				3000	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_5x16__avx_broadcast);
				3001	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x16__avx_broadcast);
				3002	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x16__avx_broadcast);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3003	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_avx_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3004	xnn_params.f32.gemm.mr = 5;
				3005	xnn_params.f32.gemm.nr = 16;
Marat Dukhan	1025ea3	2019-11-21 16:01:08 -0800	[diff] [blame]	3006	} else {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3007	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__sse_load1);
				3008	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__sse_load1);
				3009	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__sse_load1);
				3010	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__sse_load1);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3011	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_sse_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3012	xnn_params.f32.gemm.mr = 4;
				3013	xnn_params.f32.gemm.nr = 8;
Marat Dukhan	1025ea3	2019-11-21 16:01:08 -0800	[diff] [blame]	3014	}
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3015	xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2c4__sse);
				3016	xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2c4__sse);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3017	xnn_params.f32.gemm2.init.f32 = xnn_init_f32_minmax_sse_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3018	xnn_params.f32.gemm2.mr = 4;
				3019	xnn_params.f32.gemm2.nr = 2;
				3020	xnn_params.f32.gemm2.log2_kr = 2;
				3021
Marat Dukhan	479f87e	2019-11-27 15:17:06 -0800	[diff] [blame]	3022	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3023	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x3__avx512f;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3024	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3025	xnn_params.f32.dwconv[0].channel_tile = 16;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3026	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3027
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3028	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x4__avx512f;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3029	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3030	xnn_params.f32.dwconv[1].channel_tile = 16;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3031	xnn_params.f32.dwconv[1].primary_tile = 4;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3032
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3033	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x9__avx512f;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3034	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3035	xnn_params.f32.dwconv[2].channel_tile = 16;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3036	xnn_params.f32.dwconv[2].primary_tile = 9;
				3037
				3038	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x25__avx512f;
				3039	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_scalar_params;
				3040	xnn_params.f32.dwconv[3].channel_tile = 16;
				3041	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	479f87e	2019-11-27 15:17:06 -0800	[diff] [blame]	3042	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_fma3()) {
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3043	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x3__fma3;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3044	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_avx_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3045	xnn_params.f32.dwconv[0].channel_tile = 16;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3046	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3047
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3048	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x4__fma3;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3049	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_avx_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3050	xnn_params.f32.dwconv[1].channel_tile = 16;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3051	xnn_params.f32.dwconv[1].primary_tile = 4;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3052
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3053	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x9__fma3;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3054	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_avx_params;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3055	xnn_params.f32.dwconv[2].channel_tile = 16;
				3056	xnn_params.f32.dwconv[2].primary_tile = 9;
				3057
				3058	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x25__fma3;
				3059	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_avx_params;
				3060	xnn_params.f32.dwconv[3].channel_tile = 8;
				3061	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	17ec5f3	2019-11-22 13:34:16 -0800	[diff] [blame]	3062	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3063	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x3__avx;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3064	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_avx_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3065	xnn_params.f32.dwconv[0].channel_tile = 16;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3066	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3067
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3068	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x4__avx;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3069	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_avx_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3070	xnn_params.f32.dwconv[1].channel_tile = 16;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3071	xnn_params.f32.dwconv[1].primary_tile = 4;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3072
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3073	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x9__avx;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3074	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_avx_params;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3075	xnn_params.f32.dwconv[2].channel_tile = 16;
				3076	xnn_params.f32.dwconv[2].primary_tile = 9;
				3077
				3078	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x25__avx;
				3079	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_avx_params;
				3080	xnn_params.f32.dwconv[3].channel_tile = 8;
				3081	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	17ec5f3	2019-11-22 13:34:16 -0800	[diff] [blame]	3082	} else {
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3083	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x3__sse;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3084	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_sse_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3085	xnn_params.f32.dwconv[0].channel_tile = 8;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3086	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3087
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3088	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x4__sse;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3089	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_sse_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3090	xnn_params.f32.dwconv[1].channel_tile = 8;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3091	xnn_params.f32.dwconv[1].primary_tile = 4;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3092
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3093	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__sse;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3094	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_sse_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3095	xnn_params.f32.dwconv[2].channel_tile = 8;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3096	xnn_params.f32.dwconv[2].primary_tile = 9;
				3097
				3098	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x25__sse;
				3099	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_sse_params;
				3100	xnn_params.f32.dwconv[3].channel_tile = 8;
				3101	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	17ec5f3	2019-11-22 13:34:16 -0800	[diff] [blame]	3102	}
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3103	xnn_params.f32.avgpool = (struct avgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	3104	.up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__sse_c4,
				3105	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__sse_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3106	.mr = 9,
				3107	.qr = 8,
				3108	};
				3109	xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	3110	.up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__sse_c4,
				3111	.mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__sse_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3112	.mr = 9,
				3113	.qr = 8,
				3114	};
				3115	xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	3116	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__sse_c4,
				3117	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__sse_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3118	.mr = 7,
				3119	};
				3120	xnn_params.f32.maxpool = (struct maxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	3121	.ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__sse_c4,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	3122	.init.f32 = xnn_init_f32_minmax_sse_params,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3123	.mr = 9,
				3124	.qr = 8,
				3125	};
				3126	xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	3127	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__sse2_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3128	.mr = 4,
				3129	};
				3130	xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	3131	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__sse2_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3132	.mr = 9,
				3133	};
				3134	xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	3135	.mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__sse2_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3136	.mr = 9,
				3137	.qr = 8,
				3138	};
Marat Dukhan	660fd19	2020-03-10 04:55:30 -0700	[diff] [blame]	3139	xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
				3140	.ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__sse_c8,
Marat Dukhan	6972249	2019-11-11 19:55:50 -0800	[diff] [blame]	3141	.pixel_tile = 1,
				3142	.channel_tile = 8,
				3143	};
Marat Dukhan	e2c3f29	2019-11-27 15:40:54 -0800	[diff] [blame]	3144	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	3145	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__avx512f_x16;
				3146	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3147	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__avx_x16;
				3148	} else {
				3149	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__sse_x8;
				3150	}
				3151	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	3152	xnn_params.f32.clamp = (struct vunary_parameters) {
				3153	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__avx512f_x16,
				3154	.init.f32_minmax = xnn_init_f32_minmax_scalar_params,
				3155	.element_tile = 16,
				3156	};
Marat Dukhan	e2c3f29	2019-11-27 15:40:54 -0800	[diff] [blame]	3157	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	3158	xnn_params.f32.clamp = (struct vunary_parameters) {
				3159	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__avx_x16,
				3160	.init.f32_minmax = xnn_init_f32_minmax_avx_params,
				3161	.element_tile = 16,
				3162	};
Marat Dukhan	e2c3f29	2019-11-27 15:40:54 -0800	[diff] [blame]	3163	} else {
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	3164	xnn_params.f32.clamp = (struct vunary_parameters) {
				3165	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__sse_x8,
				3166	.init.f32_minmax = xnn_init_f32_minmax_sse_params,
				3167	.element_tile = 8,
				3168	};
Marat Dukhan	e2c3f29	2019-11-27 15:40:54 -0800	[diff] [blame]	3169	}
Marat Dukhan	662faa0	2019-12-09 22:48:16 -0800	[diff] [blame]	3170	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	b6bd4bc	2020-12-01 17:01:40 -0800	[diff] [blame]	3171	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__avx512f_rr1_lut16_p3_perm_x64;
				3172	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx2()) {
				3173	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__avx2_rr1_lut4_p4_perm_x56;
				3174	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3175	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__avx_rr2_lut4_p4_perm_x32;
				3176	} else {
				3177	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__sse2_rr2_lut16_p3_x12;
				3178	}
				3179	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3180	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__avx512f_x16;
Marat Dukhan	662faa0	2019-12-09 22:48:16 -0800	[diff] [blame]	3181	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_fma3()) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3182	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__fma3_x16;
Marat Dukhan	662faa0	2019-12-09 22:48:16 -0800	[diff] [blame]	3183	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3184	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__avx_x16;
Marat Dukhan	662faa0	2019-12-09 22:48:16 -0800	[diff] [blame]	3185	} else {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3186	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__sse_x8;
Marat Dukhan	662faa0	2019-12-09 22:48:16 -0800	[diff] [blame]	3187	}
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	3188	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	2881333	2020-06-10 18:05:38 -0700	[diff] [blame]	3189	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__avx512f_x16;
				3190	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3191	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__avx_x16;
Marat Dukhan	0d3f467	2020-06-25 16:42:58 -0700	[diff] [blame]	3192	} else if (cpuinfo_has_x86_sse4_1()) {
				3193	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__sse41_x8;
Marat Dukhan	2881333	2020-06-10 18:05:38 -0700	[diff] [blame]	3194	} else {
				3195	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__sse_x8;
				3196	}
				3197	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	3198	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__avx512f_x16;
				3199	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3200	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__avx_x16;
				3201	} else {
				3202	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__sse_x8;
				3203	}
Marat Dukhan	64e5251	2020-06-09 13:41:16 -0700	[diff] [blame]	3204	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
				3205	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__avx512f_x16;
				3206	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__avx512f_x16;
				3207	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__avx512f_x16;
				3208	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__avx512f_x16;
				3209	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3210	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__avx_x16;
				3211	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__avx_x16;
				3212	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__avx_x16;
				3213	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__avx_x16;
				3214	} else if (cpuinfo_has_x86_sse4_1()) {
				3215	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__sse41_x8;
				3216	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__sse41_x8;
				3217	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__sse41_x8;
				3218	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__sse41_x8;
				3219	} else {
				3220	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__sse2_x8;
				3221	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__sse2_x8;
				3222	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__sse2_x8;
				3223	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__sse2_x8;
				3224	}
Marat Dukhan	d9ca7e6	2020-09-23 23:45:29 -0700	[diff] [blame]	3225	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3226	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__avx512f_rr2_lut32_p2_perm2_scalef_div_x64;
Marat Dukhan	d9ca7e6	2020-09-23 23:45:29 -0700	[diff] [blame]	3227	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx2()) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3228	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__avx2_rr1_p5_div_x40;
T.J. Alumbaugh	dc2b29c	2020-10-14 13:56:08 -0700	[diff] [blame]	3229	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3230	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__avx_rr2_p5_nr2_x40;
Marat Dukhan	6dd7136	2020-09-17 23:11:21 -0700	[diff] [blame]	3231	} else if (cpuinfo_has_x86_sse4_1()) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3232	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__sse41_lut64_p2_div_x8;
Marat Dukhan	fa0a432	2020-01-06 16:14:29 -0800	[diff] [blame]	3233	} else {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3234	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__sse2_lut64_p2_div_x8;
Marat Dukhan	fa0a432	2020-01-06 16:14:29 -0800	[diff] [blame]	3235	}
Marat Dukhan	90eca0a	2020-03-11 00:52:23 -0700	[diff] [blame]	3236	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	3237	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__avx512f_x16;
				3238	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3239	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__avx_x16;
				3240	} else {
				3241	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__sse_x8;
				3242	}
Marat Dukhan	6804bbd	2020-06-30 19:26:11 -0700	[diff] [blame]	3243	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3244	xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__avx_sqrt_x8;
				3245	} else {
				3246	xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__sse_sqrt_x4;
				3247	}
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	3248	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan	90eca0a	2020-03-11 00:52:23 -0700	[diff] [blame]	3249	xnn_params.f32.prelu = (struct prelu_parameters) {
				3250	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__avx512f_2x16,
				3251	.row_tile = 2,
				3252	.channel_tile = 16,
				3253	};
				3254	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3255	xnn_params.f32.prelu = (struct prelu_parameters) {
				3256	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__avx_2x16,
				3257	.row_tile = 2,
				3258	.channel_tile = 16,
				3259	};
Marat Dukhan	39b5e94	2020-06-24 15:03:48 -0700	[diff] [blame]	3260	} else if (cpuinfo_has_x86_sse4_1()) {
				3261	xnn_params.f32.prelu = (struct prelu_parameters) {
				3262	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__sse41_2x8,
				3263	.row_tile = 2,
				3264	.channel_tile = 8,
				3265	};
Marat Dukhan	90eca0a	2020-03-11 00:52:23 -0700	[diff] [blame]	3266	} else {
				3267	xnn_params.f32.prelu = (struct prelu_parameters) {
				3268	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__sse2_2x8,
				3269	.row_tile = 2,
				3270	.channel_tile = 8,
				3271	};
				3272	}
Marat Dukhan	1edc454	2020-01-27 12:40:13 -0800	[diff] [blame]	3273	xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__sse2_p5_x20_acc2;
				3274	xnn_params.f32.rmax = xnn_f32_rmax_ukernel__sse;
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3275	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
				3276	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3277	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__avx512f_x32,
				3278	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__avx512f_x32,
				3279	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__avx512f_x32,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3280	.element_tile = 32,
				3281	};
				3282	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3283	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__avx512f_x32,
				3284	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__avx512f_x32,
				3285	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__avx512f_x32,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3286	.element_tile = 32,
				3287	};
				3288	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3289	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__avx512f_x32,
				3290	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__avx512f_x32,
				3291	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__avx512f_x32,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3292	.element_tile = 32,
				3293	};
				3294	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3295	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__avx512f_x32,
				3296	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__avx512f_x32,
				3297	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__avx512f_x32,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3298	.element_tile = 32,
				3299	};
				3300	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3301	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__avx512f_x32,
				3302	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__avx512f_x32,
				3303	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__avx512f_x32,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3304	.element_tile = 32,
				3305	};
				3306	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3307	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__avx512f_x32,
				3308	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__avx512f_x32,
				3309	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__avx512f_x32,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3310	.element_tile = 32,
				3311	};
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	3312	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3313	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__avx512f_x32,
				3314	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__avx512f_x32,
				3315	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__avx512f_x32,
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	3316	.element_tile = 32,
				3317	};
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3318	} else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
				3319	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3320	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__avx_x16,
				3321	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__avx_x16,
				3322	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__avx_x16,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3323	.element_tile = 16,
				3324	};
				3325	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3326	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__avx_x16,
				3327	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__avx_x16,
				3328	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__avx_x16,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3329	.element_tile = 16,
				3330	};
				3331	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3332	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__avx_x16,
				3333	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__avx_x16,
				3334	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__avx_x16,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3335	.element_tile = 16,
				3336	};
				3337	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3338	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__avx_x16,
				3339	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__avx_x16,
				3340	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__avx_x16,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3341	.element_tile = 16,
				3342	};
				3343	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3344	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__avx_x16,
				3345	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__avx_x16,
				3346	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__avx_x16,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3347	.element_tile = 16,
				3348	};
				3349	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3350	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__avx_x16,
				3351	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__avx_x16,
				3352	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__avx_x16,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3353	.element_tile = 16,
				3354	};
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	3355	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3356	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__avx_x16,
				3357	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__avx_x16,
				3358	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__avx_x16,
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	3359	.element_tile = 16,
				3360	};
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3361	} else {
				3362	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3363	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__sse_x8,
				3364	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__sse_x8,
				3365	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__sse_x8,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3366	.element_tile = 8,
				3367	};
				3368	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3369	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__sse_x8,
				3370	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__sse_x8,
				3371	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__sse_x8,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3372	.element_tile = 8,
				3373	};
				3374	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3375	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__sse_x8,
				3376	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__sse_x8,
				3377	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__sse_x8,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3378	.element_tile = 8,
				3379	};
				3380	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3381	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__sse_x8,
				3382	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__sse_x8,
				3383	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__sse_x8,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3384	.element_tile = 8,
				3385	};
				3386	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3387	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__sse_x8,
				3388	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__sse_x8,
				3389	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__sse_x8,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3390	.element_tile = 8,
				3391	};
				3392	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3393	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__sse_x8,
				3394	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__sse_x8,
				3395	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__sse_x8,
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3396	.element_tile = 8,
				3397	};
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	3398	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	3399	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__sse_x8,
				3400	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__sse_x8,
				3401	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__sse_x8,
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	3402	.element_tile = 8,
				3403	};
Marat Dukhan	9a88efe	2019-12-10 15:54:24 -0800	[diff] [blame]	3404	}
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3405	xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan	9531e9f	2020-07-24 15:25:02 -0700	[diff] [blame]	3406	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__sse_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	3407	.init.f32 = xnn_init_f32_minmax_sse_params,
Marat Dukhan	49e6ee9	2019-11-06 15:55:29 -0800	[diff] [blame]	3408	.channel_tile = 4,
				3409	.row_tile = 2,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3410	};
Marat Dukhan	efc47b8	2019-11-18 09:25:38 -0800	[diff] [blame]	3411	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	cfbed0a	2020-12-08 10:01:51 -0800	[diff] [blame]	3412	// Sparse microkernels on x86 currently target only SSE, and on processors
				3413	// with AVX ISA dense inference is expected to be faster than sparse.
				3414	if (!cpuinfo_has_x86_avx()) {
				3415	init_flags \|= XNN_INIT_FLAG_CHW_OPT;
				3416	}
				3417
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3418	xnn_params.f32.spmm = (struct spmm_parameters) {
Frank Barchard	4fd38b2	2020-10-30 17:10:11 -0700	[diff] [blame]	3419	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_32x1__sse,
				3420	.mr = 32,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3421	.nr = 1,
				3422	};
Erich Elsen	5b2e07a	2020-06-09 03:27:59 -0700	[diff] [blame]	3423	xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
				3424	.ukernel_with_symm_padding =
				3425	(xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__sse_2x2,
				3426	.output_channel_tile = 4,
				3427	.output_height_tile = 2,
				3428	.output_width_tile = 2,
				3429	};
Frank Barchard	0b18cb3	2020-11-23 10:50:44 -0800	[diff] [blame]	3430	if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_ssse3()) {
				3431	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
				3432	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__ssse3_2x4_acc2,
Frank Barchard	0b18cb3	2020-11-23 10:50:44 -0800	[diff] [blame]	3433	.output_width_tile = 4,
				3434	.output_height_tile = 2,
				3435	};
				3436	} else {
				3437	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
				3438	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__sse_2x4_acc2,
Frank Barchard	0b18cb3	2020-11-23 10:50:44 -0800	[diff] [blame]	3439	.output_width_tile = 4,
				3440	.output_height_tile = 2,
				3441	};
				3442	}
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	3443	xnn_params.f32.dwconv2d_chw_3x3s2 = (struct dwconv2d_chw_parameters) {
				3444	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__sse_1x4_acc3,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3445	.output_width_tile = 4,
				3446	.output_height_tile = 1,
				3447	};
Marat Dukhan	d050389	2020-10-30 08:22:04 -0700	[diff] [blame]	3448	xnn_params.f32.dwconv2d_chw_5x5 = (struct dwconv2d_chw_parameters) {
				3449	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5p2__sse_4x4,
Marat Dukhan	d050389	2020-10-30 08:22:04 -0700	[diff] [blame]	3450	.output_width_tile = 4,
				3451	.output_height_tile = 4,
				3452	};
Marat Dukhan	ccca214	2020-10-30 17:32:45 -0700	[diff] [blame]	3453	xnn_params.f32.dwconv2d_chw_5x5s2 = (struct dwconv2d_chw_parameters) {
				3454	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__sse_2x4,
Marat Dukhan	ccca214	2020-10-30 17:32:45 -0700	[diff] [blame]	3455	.output_width_tile = 4,
				3456	.output_height_tile = 2,
				3457	};
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	3458	xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
				3459	.ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__sse_x4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3460	.channel_tile = 4,
				3461	};
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	3462	xnn_params.f32.ibilinear_chw = (struct ibilinear_chw_parameters) {
Artsiom Ablavatski	b3ffd58	2021-03-31 13:00:08 -0700	[diff] [blame]	3463	.ukernel = (xnn_ibilinear_chw_ukernel_function) xnn_f32_ibilinear_chw_ukernel__sse_p8,
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	3464	.channel_tile = 1,
Artsiom Ablavatski	b3ffd58	2021-03-31 13:00:08 -0700	[diff] [blame]	3465	.pixel_tile = 8,
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	3466	};
Marat Dukhan	efc47b8	2019-11-18 09:25:38 -0800	[diff] [blame]	3467	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3468	#endif // XNN_NO_F32_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	3469
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	3470	/************************* VCVT micro-kernels *************************/
				3471	#ifndef XNN_NO_VCVT_OPERATORS
				3472	init_flags \|= XNN_INIT_FLAG_VCVT;
				3473
				3474	if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
				3475	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__avx512skx_x16;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	3476	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__avx512skx_x16;
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	3477	} else if (cpuinfo_has_x86_f16c()) {
				3478	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__f16c_x16;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	3479	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__f16c_x16;
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	3480	} else if (cpuinfo_has_x86_avx()) {
				3481	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__avx_int16_x16;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	3482	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__avx_x24;
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	3483	} else if (cpuinfo_has_x86_sse4_1()) {
				3484	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__sse41_int16_x16;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	3485	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__sse41_x8;
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	3486	} else {
				3487	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__sse2_int16_x32;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	3488	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__sse2_x16;
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	3489	}
Marat Dukhan	ed2d776	2021-12-03 23:51:19 -0800	[diff] [blame]	3490	if (cpuinfo_has_x86_sse4_1()) {
				3491	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
				3492	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qs8_vcvt_ukernel__sse41_x32,
				3493	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_sse4_params,
				3494	.element_tile = 32,
				3495	};
				3496	} else {
				3497	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
				3498	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qs8_vcvt_ukernel__sse2_x32,
				3499	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_sse2_params,
				3500	.element_tile = 32,
				3501	};
				3502	}
				3503	xnn_params.vcvt.f32_to_qu8 = (struct vunary_parameters) {
				3504	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__sse2_x32,
				3505	.init.f32_qu8_cvt = xnn_init_f32_qu8_cvt_sse2_params,
				3506	.element_tile = 32,
				3507	};
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	3508	#endif // XNN_NO_VCVT_OPERATORS
				3509
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	3510	/************************** X32 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3511	#ifndef XNN_NO_X32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	3512	init_flags \|= XNN_INIT_FLAG_X32;
				3513
Marat Dukhan	57dccd8	2020-04-14 00:53:10 -0700	[diff] [blame]	3514	xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__sse2;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3515	xnn_params.x32.zip = (struct zip_parameters) {
				3516	.x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__sse2,
				3517	.x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__sse2,
				3518	.x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__sse2,
				3519	.xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__sse2,
				3520	};
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	3521	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	ad71b9a	2020-11-20 00:01:51 -0800	[diff] [blame]	3522	xnn_params.x32.depthtospace2d_chw2hwc = (struct depthtospace2d_chw2hwc_parameters) {
				3523	.ukernel = (xnn_depthtospace2d_chw2hwc_ukernel_function) xnn_x32_depthtospace2d_chw2hwc_ukernel__scalar,
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	3524	.channel_tile = 1,
				3525	.pixel_tile = 1,
				3526	};
				3527	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3528	#endif // XNN_NO_X32_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	3529
Marat Dukhan	048931b	2020-11-24 20:53:54 -0800	[diff] [blame]	3530	/************************** XX micro-kernels **************************/
				3531	#ifndef XNN_NO_XX_OPERATORS
				3532	init_flags \|= XNN_INIT_FLAG_XX;
				3533
				3534	xnn_params.xx.copy = (xnn_univector_ukernel_function) xnn_xx_copy_ukernel__memcpy;
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	3535	xnn_params.xx.fill = (struct fill_parameters) {
				3536	.ukernel = (xnn_fill_ukernel_function) xnn_xx_fill_ukernel__sse2_x64,
				3537	.row_tile = 1,
				3538	};
Marat Dukhan	0461f2d	2021-08-08 12:36:29 -0700	[diff] [blame]	3539	xnn_params.xx.pad = (struct pad_parameters) {
				3540	.ukernel = (xnn_pad_ukernel_function) xnn_xx_pad_ukernel__sse2,
				3541	.row_tile = 1,
				3542	};
Marat Dukhan	048931b	2020-11-24 20:53:54 -0800	[diff] [blame]	3543	#endif
				3544
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	3545	#elif XNN_ARCH_WASMSIMD
				3546
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	3547	/************************** QC8 micro-kernels **************************/
				3548	#ifndef XNN_NO_QS8_OPERATORS
				3549	init_flags \|= XNN_INIT_FLAG_QC8;
				3550
Marat Dukhan	189c1d0	2021-09-03 15:39:54 -0700	[diff] [blame]	3551	#if defined(XNN_WASMSIMD_VERSION) && (XNN_WASMSIMD_VERSION >= 88)
				3552	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_3x4c2__wasmsimd_dot16x2_ld128);
				3553	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_3x4c2__wasmsimd_dot16x2_ld128);
				3554	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x4c2__wasmsimd_dot16x2_ld128);
				3555	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x4c2__wasmsimd_dot16x2_ld128);
				3556	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_wasmsimd_params;
				3557	xnn_params.qc8.gemm.mr = 3;
				3558	xnn_params.qc8.gemm.nr = 4;
				3559	xnn_params.qc8.gemm.log2_kr = 1;
				3560	#else
				3561	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_3x4c8__wasmsimd_mul16_ld64);
				3562	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_3x4c8__wasmsimd_mul16_ld64);
				3563	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x4c8__wasmsimd_mul16_ld64);
				3564	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x4c8__wasmsimd_mul16_ld64);
				3565	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_wasmsimd_params;
				3566	xnn_params.qc8.gemm.mr = 3;
				3567	xnn_params.qc8.gemm.nr = 4;
				3568	xnn_params.qc8.gemm.log2_kr = 3;
				3569	#endif
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	3570
Marat Dukhan	9cedb59	2021-08-17 17:25:24 -0700	[diff] [blame]	3571	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__wasmsimd_mul16_add16;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	3572	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_wasmsimd_params;
Marat Dukhan	9cedb59	2021-08-17 17:25:24 -0700	[diff] [blame]	3573	xnn_params.qc8.dwconv[0].channel_tile = 16;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	3574	xnn_params.qc8.dwconv[0].primary_tile = 9;
Marat Dukhan	9cedb59	2021-08-17 17:25:24 -0700	[diff] [blame]	3575	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__wasmsimd_mul16_add16;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	3576	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_wasmsimd_params;
Marat Dukhan	9cedb59	2021-08-17 17:25:24 -0700	[diff] [blame]	3577	xnn_params.qc8.dwconv[1].channel_tile = 16;
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	3578	xnn_params.qc8.dwconv[1].primary_tile = 25;
				3579	#endif // XNN_NO_QC8_OPERATORS
				3580
Marat Dukhan	07e5040	2020-08-05 17:16:53 -0700	[diff] [blame]	3581	/************************** QS8 micro-kernels **************************/
				3582	#ifndef XNN_NO_QS8_OPERATORS
				3583	init_flags \|= XNN_INIT_FLAG_QS8;
				3584
Marat Dukhan	189c1d0	2021-09-03 15:39:54 -0700	[diff] [blame]	3585	#if defined(XNN_WASMSIMD_VERSION) && (XNN_WASMSIMD_VERSION >= 88)
				3586	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__wasmsimd_dot16x2_ld128);
				3587	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__wasmsimd_dot16x2_ld128);
				3588	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__wasmsimd_dot16x2_ld128);
				3589	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__wasmsimd_dot16x2_ld128);
				3590	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_wasmsimd_params;
				3591	xnn_params.qs8.gemm.mr = 3;
				3592	xnn_params.qs8.gemm.nr = 4;
				3593	xnn_params.qs8.gemm.log2_kr = 1;
				3594	#else // XNN_WASMSIMD_VERSION >= 88
				3595	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__wasmsimd_mul16_ld64);
				3596	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__wasmsimd_mul16_ld64);
				3597	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__wasmsimd_mul16_ld64);
				3598	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__wasmsimd_mul16_ld64);
				3599	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_wasmsimd_params;
				3600	xnn_params.qs8.gemm.mr = 3;
				3601	xnn_params.qs8.gemm.nr = 4;
				3602	xnn_params.qs8.gemm.log2_kr = 3;
				3603	#endif // XNN_WASMSIMD_VERSION >= 88
Marat Dukhan	07e5040	2020-08-05 17:16:53 -0700	[diff] [blame]	3604
Marat Dukhan	9cedb59	2021-08-17 17:25:24 -0700	[diff] [blame]	3605	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__wasmsimd_mul16_add16;
Marat Dukhan	400e7cb	2021-08-07 15:14:54 -0700	[diff] [blame]	3606	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_wasmsimd_params;
Marat Dukhan	9cedb59	2021-08-17 17:25:24 -0700	[diff] [blame]	3607	xnn_params.qs8.dwconv[0].channel_tile = 16;
Marat Dukhan	07e5040	2020-08-05 17:16:53 -0700	[diff] [blame]	3608	xnn_params.qs8.dwconv[0].primary_tile = 9;
Marat Dukhan	9cedb59	2021-08-17 17:25:24 -0700	[diff] [blame]	3609	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up16x25__wasmsimd_mul16_add16;
Marat Dukhan	400e7cb	2021-08-07 15:14:54 -0700	[diff] [blame]	3610	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_wasmsimd_params;
Marat Dukhan	9cedb59	2021-08-17 17:25:24 -0700	[diff] [blame]	3611	xnn_params.qs8.dwconv[1].channel_tile = 16;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	3612	xnn_params.qs8.dwconv[1].primary_tile = 25;
Marat Dukhan	9e0b539	2020-08-07 02:29:34 -0700	[diff] [blame]	3613
				3614	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				3615	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__wasmsimd_c8_acc2,
				3616	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__wasmsimd_c8_acc2,
				3617	.mr = 7,
				3618	};
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	3619
				3620	xnn_params.qs8.vadd = (struct vbinary_parameters) {
Marat Dukhan	e20a873	2021-12-07 17:11:37 -0800	[diff] [blame^]	3621	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__wasmsimd_x32,
				3622	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__wasmsimd_x32,
				3623	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__wasmsimd_x32,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	3624	.init.qs8_addsub = xnn_init_qs8_add_minmax_wasmsimd_params,
Marat Dukhan	e20a873	2021-12-07 17:11:37 -0800	[diff] [blame^]	3625	.element_tile = 32,
Marat Dukhan	ff20948	2020-09-03 14:26:53 -0700	[diff] [blame]	3626	};
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	3627	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				3628	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__wasmsimd_mul32_ld64_x8,
				3629	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__wasmsimd_mul32_ld64_x8,
				3630	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__wasmsimd_mul32_ld64_x8,
				3631	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_wasmsimd_params,
				3632	.element_tile = 8,
				3633	};
Marat Dukhan	07e5040	2020-08-05 17:16:53 -0700	[diff] [blame]	3634	#endif // XNN_NO_QS8_OPERATORS
				3635
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	3636	/************************** QU8 micro-kernels **************************/
				3637	#ifndef XNN_NO_QU8_OPERATORS
				3638	init_flags \|= XNN_INIT_FLAG_QU8;
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	3639
Marat Dukhan	189c1d0	2021-09-03 15:39:54 -0700	[diff] [blame]	3640	#if defined(XNN_WASMSIMD_VERSION) && (XNN_WASMSIMD_VERSION >= 88)
				3641	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_3x4c2__wasmsimd_dot16x2_ld128);
				3642	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_3x4c2__wasmsimd_dot16x2_ld128);
				3643	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c2__wasmsimd_dot16x2_ld128);
				3644	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c2__wasmsimd_dot16x2_ld128);
				3645	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_wasmsimd_params;
				3646	xnn_params.qu8.gemm.mr = 3;
				3647	xnn_params.qu8.gemm.nr = 4;
				3648	xnn_params.qu8.gemm.log2_kr = 1;
				3649	#else // XNN_WASMSIMD_VERSION >= 88
				3650	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_3x4c8__wasmsimd_mul32_ld64);
				3651	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_3x4c8__wasmsimd_mul32_ld64);
				3652	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__wasmsimd_mul32_ld64);
				3653	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__wasmsimd_mul32_ld64);
				3654	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_wasmsimd_params;
				3655	xnn_params.qu8.gemm.mr = 3;
				3656	xnn_params.qu8.gemm.nr = 4;
				3657	xnn_params.qu8.gemm.log2_kr = 3;
				3658	#endif // XNN_WASMSIMD_VERSION >= 88
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3659
Marat Dukhan	a97e975	2021-07-15 16:30:41 -0700	[diff] [blame]	3660	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up8x9__wasmsimd_mul16;
				3661	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_wasmsimd_params;
				3662	xnn_params.qu8.dwconv[0].channel_tile = 8;
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	3663	xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhan	a97e975	2021-07-15 16:30:41 -0700	[diff] [blame]	3664	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up8x25__wasmsimd_mul16;
				3665	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_wasmsimd_params;
				3666	xnn_params.qu8.dwconv[1].channel_tile = 8;
				3667	xnn_params.qu8.dwconv[1].primary_tile = 25;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3668
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	3669	xnn_params.qu8.avgpool = (struct avgpool_parameters) {
				3670	.up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__scalar_c1,
				3671	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3672	.mr = 9,
				3673	.qr = 8,
				3674	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	3675	xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
				3676	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__scalar_c1,
				3677	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3678	.mr = 7,
				3679	};
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	3680
				3681	xnn_params.qu8.vadd = (struct vbinary_parameters) {
Marat Dukhan	e20a873	2021-12-07 17:11:37 -0800	[diff] [blame^]	3682	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__wasmsimd_x32,
				3683	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__wasmsimd_x32,
				3684	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__wasmsimd_x32,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	3685	.init.qu8_addsub = xnn_init_qu8_add_minmax_wasmsimd_params,
Marat Dukhan	e20a873	2021-12-07 17:11:37 -0800	[diff] [blame^]	3686	.element_tile = 32,
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	3687	};
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	3688	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				3689	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__wasmsimd_mul32_ld64_x8,
				3690	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__wasmsimd_mul32_ld64_x8,
				3691	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__wasmsimd_mul32_ld64_x8,
				3692	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_wasmsimd_params,
				3693	.element_tile = 8,
				3694	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	3695	#endif // XNN_NO_QU8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	3696
Marat Dukhan	dc5c148	2021-08-16 09:03:15 -0700	[diff] [blame]	3697	/************************** S8 micro-kernels **************************/
				3698	#ifndef XNN_NO_S8_OPERATORS
				3699	init_flags \|= XNN_INIT_FLAG_S8;
				3700
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	3701	xnn_params.s8.clamp = (struct vunary_parameters) {
				3702	.ukernel = (xnn_univector_ukernel_function) xnn_s8_vclamp_ukernel__wasmsimd_x64,
				3703	.init.s8_minmax = xnn_init_s8_minmax_wasmsimd_params,
				3704	.element_tile = 64,
				3705	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	3706	#if defined(XNN_WASMSIMD_VERSION) && (XNN_WASMSIMD_VERSION >= 88)
				3707	xnn_params.s8.ibilinear = (struct ibilinear_parameters) {
				3708	.ukernel = (xnn_ibilinear_ukernel_function) xnn_s8_ibilinear_ukernel__wasmsimd_dot16x2_c8,
				3709	.pixel_tile = 1,
				3710	.channel_tile = 8,
				3711	};
				3712	#else // XNN_WASMSIMD_VERSION >= 88
				3713	xnn_params.s8.ibilinear = (struct ibilinear_parameters) {
				3714	.ukernel = (xnn_ibilinear_ukernel_function) xnn_s8_ibilinear_ukernel__wasmsimd_mul32_c8,
				3715	.pixel_tile = 1,
				3716	.channel_tile = 8,
				3717	};
				3718	#endif // XNN_WASMSIMD_VERSION >= 88
Marat Dukhan	dc5c148	2021-08-16 09:03:15 -0700	[diff] [blame]	3719	xnn_params.s8.maxpool = (struct maxpool_parameters) {
				3720	.ukernel = (xnn_maxpool_ukernel_function) xnn_s8_maxpool_minmax_ukernel_9p8x__wasmsimd_c16,
				3721	.init.s8 = xnn_init_s8_minmax_wasmsimd_params,
				3722	.mr = 9,
				3723	.qr = 8,
				3724	};
				3725	#endif // XNN_NO_S8_OPERATORS
				3726
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	3727	/************************** U8 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3728	#ifndef XNN_NO_U8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	3729	init_flags \|= XNN_INIT_FLAG_U8;
				3730
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	3731	xnn_params.u8.clamp = (struct vunary_parameters) {
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	3732	.ukernel = (xnn_univector_ukernel_function) xnn_u8_vclamp_ukernel__wasmsimd_x64,
				3733	.init.u8_minmax = xnn_init_u8_minmax_wasmsimd_params,
				3734	.element_tile = 64,
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	3735	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	3736	#if defined(XNN_WASMSIMD_VERSION) && (XNN_WASMSIMD_VERSION >= 88)
				3737	xnn_params.u8.ibilinear = (struct ibilinear_parameters) {
				3738	.ukernel = (xnn_ibilinear_ukernel_function) xnn_u8_ibilinear_ukernel__wasmsimd_dot16x2_c8,
				3739	.pixel_tile = 1,
				3740	.channel_tile = 8,
				3741	};
				3742	#else // XNN_WASMSIMD_VERSION >= 88
				3743	xnn_params.u8.ibilinear = (struct ibilinear_parameters) {
				3744	.ukernel = (xnn_ibilinear_ukernel_function) xnn_u8_ibilinear_ukernel__wasmsimd_mul32_c8,
				3745	.pixel_tile = 1,
				3746	.channel_tile = 8,
				3747	};
				3748	#endif // XNN_WASMSIMD_VERSION >= 88
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3749	xnn_params.u8.maxpool = (struct maxpool_parameters) {
Marat Dukhan	f158942	2021-08-15 20:37:06 -0700	[diff] [blame]	3750	.ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__wasmsimd_c16,
				3751	.init.u8 = xnn_init_u8_minmax_wasmsimd_params,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3752	.mr = 9,
				3753	.qr = 8,
				3754	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3755	xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
				3756	xnn_params.u8.rmax = xnn_u8_rmax_ukernel__scalar;
				3757	#endif // XNN_NO_U8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	3758
				3759	/************************** X8 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3760	#ifndef XNN_NO_X8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	3761	init_flags \|= XNN_INIT_FLAG_X8;
				3762
Marat Dukhan	d67539d	2021-09-08 23:06:03 -0700	[diff] [blame]	3763	xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar_x4;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3764	xnn_params.x8.zip = (struct zip_parameters) {
				3765	.x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__scalar,
				3766	.x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__scalar,
				3767	.x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__scalar,
				3768	.xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__scalar,
				3769	};
				3770	#endif // XNN_NO_X8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	3771
				3772	/************************** F32 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3773	#ifndef XNN_NO_F32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	3774	init_flags \|= XNN_INIT_FLAG_F32;
				3775
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3776	if (is_wasm_x86) {
Frank Barchard	0725b8d	2020-12-07 11:07:35 -0800	[diff] [blame]	3777	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_x86_splat);
				3778	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_x86_splat);
				3779	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_x86_splat);
				3780	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__wasmsimd_x86_splat);
Marat Dukhan	688f6d8	2020-07-14 17:02:11 -0700	[diff] [blame]	3781	xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_4x8__wasmsimd_splat);
				3782	xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_4x8__wasmsimd_splat);
				3783	xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x8__wasmsimd_splat);
				3784	xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x8__wasmsimd_splat);
Marat Dukhan	802808c	2020-06-16 11:01:17 -0700	[diff] [blame]	3785	xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x8__wasmsimd_splat);
				3786	xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x8__wasmsimd_splat);
				3787	xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x8__wasmsimd_splat);
				3788	xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x8__wasmsimd_splat);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3789	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3790	xnn_params.f32.gemm.mr = 4;
				3791	xnn_params.f32.gemm.nr = 8;
Marat Dukhan	e39e646	2020-07-09 01:33:36 -0700	[diff] [blame]	3792
				3793	xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2c4__wasmsimd_x86);
				3794	xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2c4__wasmsimd_x86);
				3795	xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2c4__wasmsimd);
				3796	xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2c4__wasmsimd);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3797	xnn_params.f32.gemm2.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	e39e646	2020-07-09 01:33:36 -0700	[diff] [blame]	3798	xnn_params.f32.gemm2.mr = 4;
				3799	xnn_params.f32.gemm2.nr = 2;
				3800	xnn_params.f32.gemm2.log2_kr = 2;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3801	} else {
Frank Barchard	0725b8d	2020-12-07 11:07:35 -0800	[diff] [blame]	3802	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_arm_splat);
				3803	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_arm_splat);
				3804	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_arm_splat);
				3805	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__wasmsimd_arm_splat);
Marat Dukhan	688f6d8	2020-07-14 17:02:11 -0700	[diff] [blame]	3806	xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_5x8__wasmsimd_splat);
				3807	xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_5x8__wasmsimd_splat);
				3808	xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x8__wasmsimd_splat);
				3809	xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x8__wasmsimd_splat);
Marat Dukhan	802808c	2020-06-16 11:01:17 -0700	[diff] [blame]	3810	xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_5x8__wasmsimd_splat);
				3811	xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_5x8__wasmsimd_splat);
				3812	xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x8__wasmsimd_splat);
				3813	xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x8__wasmsimd_splat);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3814	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	1bbf96b	2020-06-15 23:01:20 -0700	[diff] [blame]	3815	xnn_params.f32.gemm.mr = 5;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3816	xnn_params.f32.gemm.nr = 8;
Marat Dukhan	e39e646	2020-07-09 01:33:36 -0700	[diff] [blame]	3817
				3818	xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2c4__wasmsimd_arm);
				3819	xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2c4__wasmsimd_arm);
				3820	xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2c4__wasmsimd);
				3821	xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2c4__wasmsimd);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3822	xnn_params.f32.gemm2.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	e39e646	2020-07-09 01:33:36 -0700	[diff] [blame]	3823	xnn_params.f32.gemm2.mr = 4;
				3824	xnn_params.f32.gemm2.nr = 2;
				3825	xnn_params.f32.gemm2.log2_kr = 2;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3826	}
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3827
Marat Dukhan	ac014d7	2020-06-16 08:36:47 -0700	[diff] [blame]	3828	if (is_wasm_x86) {
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3829	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x3__wasmsimd_x86;
				3830	xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up8x3__wasmsimd;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3831	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	ac014d7	2020-06-16 08:36:47 -0700	[diff] [blame]	3832	xnn_params.f32.dwconv[0].channel_tile = 8;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3833	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3834
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3835	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86;
				3836	xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up8x4__wasmsimd;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3837	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	ac014d7	2020-06-16 08:36:47 -0700	[diff] [blame]	3838	xnn_params.f32.dwconv[1].channel_tile = 8;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3839	xnn_params.f32.dwconv[1].primary_tile = 4;
				3840
				3841	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86;
				3842	xnn_params.f32.dwconv[2].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up8x9__wasmsimd;
				3843	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
				3844	xnn_params.f32.dwconv[2].channel_tile = 8;
				3845	xnn_params.f32.dwconv[2].primary_tile = 9;
Marat Dukhan	ac014d7	2020-06-16 08:36:47 -0700	[diff] [blame]	3846	} else {
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3847	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x3__wasmsimd_arm;
				3848	xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up4x3__wasmsimd;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3849	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	ac014d7	2020-06-16 08:36:47 -0700	[diff] [blame]	3850	xnn_params.f32.dwconv[0].channel_tile = 4;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3851	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3852
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3853	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm;
				3854	xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up4x4__wasmsimd;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	3855	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	ac014d7	2020-06-16 08:36:47 -0700	[diff] [blame]	3856	xnn_params.f32.dwconv[1].channel_tile = 4;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3857	xnn_params.f32.dwconv[1].primary_tile = 4;
				3858
				3859	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm;
				3860	xnn_params.f32.dwconv[2].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up4x9__wasmsimd;
				3861	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
				3862	xnn_params.f32.dwconv[2].channel_tile = 4;
				3863	xnn_params.f32.dwconv[2].primary_tile = 9;
Marat Dukhan	ac014d7	2020-06-16 08:36:47 -0700	[diff] [blame]	3864	}
				3865
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	3866	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm;
				3867	xnn_params.f32.dwconv[3].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up4x25__wasmsimd;
				3868	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_scalar_params;
				3869	xnn_params.f32.dwconv[3].channel_tile = 4;
				3870	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	3871
Marat Dukhan	3b7432d	2020-07-16 17:46:32 -0700	[diff] [blame]	3872	if (is_wasm_x86) {
				3873	xnn_params.f32.avgpool = (struct avgpool_parameters) {
				3874	.up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__wasmsimd_x86_c4,
				3875	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__wasmsimd_x86_c4,
				3876	.mr = 9,
				3877	.qr = 8,
				3878	};
Marat Dukhan	1483c53	2020-07-16 18:08:19 -0700	[diff] [blame]	3879	xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
				3880	.up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__wasmsimd_x86_c4,
				3881	.mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__wasmsimd_x86_c4,
				3882	.mr = 9,
				3883	.qr = 8,
				3884	};
Marat Dukhan	c601680	2020-07-16 18:51:28 -0700	[diff] [blame]	3885	xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
				3886	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__wasmsimd_x86_c4,
				3887	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__wasmsimd_x86_c4,
				3888	.mr = 7,
				3889	};
Marat Dukhan	3b7432d	2020-07-16 17:46:32 -0700	[diff] [blame]	3890	} else {
				3891	xnn_params.f32.avgpool = (struct avgpool_parameters) {
				3892	.up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__wasmsimd_arm_c4,
				3893	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__wasmsimd_arm_c4,
				3894	.mr = 9,
				3895	.qr = 8,
				3896	};
Marat Dukhan	1483c53	2020-07-16 18:08:19 -0700	[diff] [blame]	3897	xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
				3898	.up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__wasmsimd_arm_c4,
				3899	.mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__wasmsimd_arm_c4,
				3900	.mr = 9,
				3901	.qr = 8,
				3902	};
Marat Dukhan	c601680	2020-07-16 18:51:28 -0700	[diff] [blame]	3903	xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
				3904	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__wasmsimd_arm_c4,
				3905	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__wasmsimd_arm_c4,
				3906	.mr = 7,
				3907	};
Marat Dukhan	3b7432d	2020-07-16 17:46:32 -0700	[diff] [blame]	3908	}
Marat Dukhan	f6e2480	2020-07-08 22:20:40 -0700	[diff] [blame]	3909	if (is_wasm_x86) {
				3910	xnn_params.f32.maxpool = (struct maxpool_parameters) {
				3911	.ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__wasmsimd_x86_c4,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	3912	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	f6e2480	2020-07-08 22:20:40 -0700	[diff] [blame]	3913	.mr = 9,
				3914	.qr = 8,
				3915	};
				3916	} else {
				3917	xnn_params.f32.maxpool = (struct maxpool_parameters) {
				3918	.ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__wasmsimd_arm_c4,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	3919	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	f6e2480	2020-07-08 22:20:40 -0700	[diff] [blame]	3920	.mr = 9,
				3921	.qr = 8,
				3922	};
				3923	}
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3924	xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhan	40f0552	2020-07-16 22:33:12 -0700	[diff] [blame]	3925	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__wasmsimd_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3926	.mr = 4,
				3927	};
				3928	xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhan	40f0552	2020-07-16 22:33:12 -0700	[diff] [blame]	3929	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__wasmsimd_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3930	.mr = 9,
				3931	};
				3932	xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhan	40f0552	2020-07-16 22:33:12 -0700	[diff] [blame]	3933	.mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__wasmsimd_c4,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	3934	.mr = 9,
				3935	.qr = 8,
				3936	};
Marat Dukhan	660fd19	2020-03-10 04:55:30 -0700	[diff] [blame]	3937	xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
Marat Dukhan	00d1d6e	2020-07-09 01:37:27 -0700	[diff] [blame]	3938	.ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__wasmsimd_c8,
Marat Dukhan	6972249	2019-11-11 19:55:50 -0800	[diff] [blame]	3939	.pixel_tile = 1,
				3940	.channel_tile = 8,
				3941	};
Marat Dukhan	37c8351	2020-06-29 13:25:53 -0700	[diff] [blame]	3942	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__wasmsimd_x8;
Marat Dukhan	c303fe6	2020-06-26 10:09:25 -0700	[diff] [blame]	3943	if (is_wasm_x86) {
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	3944	xnn_params.f32.clamp = (struct vunary_parameters) {
				3945	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__wasmsimd_x86_x8,
				3946	.init.f32_minmax = xnn_init_f32_minmax_scalar_params,
				3947	.element_tile = 8,
				3948	};
Marat Dukhan	c303fe6	2020-06-26 10:09:25 -0700	[diff] [blame]	3949	} else {
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	3950	xnn_params.f32.clamp = (struct vunary_parameters) {
				3951	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__wasmsimd_arm_x8,
				3952	.init.f32_minmax = xnn_init_f32_minmax_scalar_params,
				3953	.element_tile = 8,
				3954	};
Marat Dukhan	c303fe6	2020-06-26 10:09:25 -0700	[diff] [blame]	3955	}
Marat Dukhan	b6bd4bc	2020-12-01 17:01:40 -0800	[diff] [blame]	3956	if (is_wasm_x86) {
				3957	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__wasmsimd_x86_rr2_p6_x20;
				3958	} else {
				3959	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__wasmsimd_arm_rr2_p6_x20;
				3960	}
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3961	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__wasmsimd_x16;
Marat Dukhan	f4935a2	2020-07-16 15:59:10 -0700	[diff] [blame]	3962	if (is_wasm_x86) {
				3963	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__wasmsimd_minmax_x8;
				3964	} else {
				3965	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__wasmsimd_bitselect_x8;
				3966	}
Marat Dukhan	37c8351	2020-06-29 13:25:53 -0700	[diff] [blame]	3967	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__wasmsimd_x8;
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3968	xnn_params.f32.relu = (xnn_univector_ukernel_function) xnn_f32_vrelu_ukernel__wasmsimd_x16;
Marat Dukhan	189c1d0	2021-09-03 15:39:54 -0700	[diff] [blame]	3969	#if defined(XNN_WASMSIMD_VERSION) && (XNN_WASMSIMD_VERSION >= 91)
				3970	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__wasmsimd_native_x8;
				3971	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__wasmsimd_native_x8;
				3972	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__wasmsimd_native_x8;
				3973	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__wasmsimd_native_x8;
				3974	#else // XNN_WASMSIMD_VERSION >= 91
				3975	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__wasmsimd_addsub_x8;
				3976	if (is_wasm_x86) {
				3977	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__wasmsimd_addsub_x8;
				3978	} else {
				3979	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__wasmsimd_cvt_x8;
				3980	}
				3981	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__wasmsimd_addsub_x8;
				3982	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__wasmsimd_addsub_x8;
				3983	#endif // XNN_WASMSIMD_VERSION >= 91
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	3984	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__wasmsimd_p5_div_x16;
Marat Dukhan	37c8351	2020-06-29 13:25:53 -0700	[diff] [blame]	3985	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__wasmsimd_x8;
Marat Dukhan	6804bbd	2020-06-30 19:26:11 -0700	[diff] [blame]	3986	xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__wasmsimd_sqrt_x8;
Marat Dukhan	195f8eb	2020-06-25 12:50:57 -0700	[diff] [blame]	3987	if (is_wasm_x86) {
				3988	xnn_params.f32.prelu = (struct prelu_parameters) {
Marat Dukhan	7829928	2020-07-15 17:38:06 -0700	[diff] [blame]	3989	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__wasmsimd_minmax_2x8,
Marat Dukhan	195f8eb	2020-06-25 12:50:57 -0700	[diff] [blame]	3990	.row_tile = 2,
				3991	.channel_tile = 8,
				3992	};
				3993	} else {
				3994	xnn_params.f32.prelu = (struct prelu_parameters) {
Marat Dukhan	7829928	2020-07-15 17:38:06 -0700	[diff] [blame]	3995	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__wasmsimd_bitselect_2x8,
Marat Dukhan	195f8eb	2020-06-25 12:50:57 -0700	[diff] [blame]	3996	.row_tile = 2,
				3997	.channel_tile = 8,
				3998	};
				3999	}
Marat Dukhan	52238f0	2020-07-16 15:30:28 -0700	[diff] [blame]	4000	xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__wasmsimd_p5_x16_acc2;
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4001	if (is_wasm_x86) {
Marat Dukhan	0bf8afa	2021-09-20 10:02:18 -0700	[diff] [blame]	4002	xnn_params.f32.rmax = xnn_f32_rmax_ukernel__wasmsimd_x86;
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4003	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4004	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__wasmsimd_x86_x16,
				4005	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasmsimd_x86_x16,
				4006	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasmsimd_x86_x16,
				4007	.linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_ukernel__wasmsimd_x16,
				4008	.linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_ukernel__wasmsimd_x16,
				4009	.linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_ukernel__wasmsimd_x16,
				4010	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4011	};
				4012	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	b392f8e	2020-10-27 10:46:44 -0700	[diff] [blame]	4013	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__wasmsimd_x86_x16,
				4014	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__wasmsimd_x86_x16,
				4015	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__wasmsimd_x86_x16,
				4016	.linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_ukernel__wasmsimd_x16,
				4017	.linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_ukernel__wasmsimd_x16,
				4018	.linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_ukernel__wasmsimd_x16,
				4019	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4020	};
				4021	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4022	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__wasmsimd_x86_x16,
				4023	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasmsimd_x86_x16,
				4024	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasmsimd_x86_x16,
				4025	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4026	};
				4027	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4028	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__wasmsimd_x86_x16,
				4029	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasmsimd_x86_x16,
				4030	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasmsimd_x86_x16,
Frank Barchard	c67dd7f	2020-07-06 11:23:57 -0700	[diff] [blame]	4031
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4032	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4033	};
				4034	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4035	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__wasmsimd_x86_x16,
				4036	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasmsimd_x86_x16,
				4037	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasmsimd_x86_x16,
				4038	.linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_ukernel__wasmsimd_x16,
				4039	.linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_ukernel__wasmsimd_x16,
				4040	.linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_ukernel__wasmsimd_x16,
				4041	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4042	};
				4043	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4044	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__wasmsimd_x86_x16,
				4045	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__wasmsimd_x86_x16,
				4046	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__wasmsimd_x86_x16,
				4047	.linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_ukernel__wasmsimd_x16,
				4048	.linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_ukernel__wasmsimd_x16,
				4049	.linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_ukernel__wasmsimd_x16,
				4050	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4051	};
				4052	} else {
Marat Dukhan	0bf8afa	2021-09-20 10:02:18 -0700	[diff] [blame]	4053	xnn_params.f32.rmax = xnn_f32_rmax_ukernel__wasmsimd_arm;
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4054	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4055	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__wasmsimd_arm_x16,
				4056	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasmsimd_arm_x16,
				4057	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasmsimd_arm_x16,
				4058	.linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_ukernel__wasmsimd_x16,
				4059	.linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_ukernel__wasmsimd_x16,
				4060	.linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_ukernel__wasmsimd_x16,
				4061	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4062	};
				4063	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	b392f8e	2020-10-27 10:46:44 -0700	[diff] [blame]	4064	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__wasmsimd_arm_x16,
				4065	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__wasmsimd_arm_x16,
				4066	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__wasmsimd_arm_x16,
				4067	.linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_ukernel__wasmsimd_x16,
				4068	.linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_ukernel__wasmsimd_x16,
				4069	.linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_ukernel__wasmsimd_x16,
				4070	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4071	};
				4072	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4073	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__wasmsimd_arm_x16,
				4074	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasmsimd_arm_x16,
				4075	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasmsimd_arm_x16,
				4076	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4077	};
				4078	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4079	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__wasmsimd_arm_x16,
				4080	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasmsimd_arm_x16,
				4081	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasmsimd_arm_x16,
				4082	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4083	};
				4084	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4085	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__wasmsimd_arm_x16,
				4086	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasmsimd_arm_x16,
				4087	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasmsimd_arm_x16,
				4088	.linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_ukernel__wasmsimd_x16,
				4089	.linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_ukernel__wasmsimd_x16,
				4090	.linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_ukernel__wasmsimd_x16,
				4091	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4092	};
				4093	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4094	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__wasmsimd_arm_x16,
				4095	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__wasmsimd_arm_x16,
				4096	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__wasmsimd_arm_x16,
				4097	.linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_ukernel__wasmsimd_x16,
				4098	.linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_ukernel__wasmsimd_x16,
				4099	.linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_ukernel__wasmsimd_x16,
				4100	.element_tile = 16,
Marat Dukhan	cdc5655	2020-06-26 19:49:41 -0700	[diff] [blame]	4101	};
				4102	}
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	4103	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4104	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__wasmsimd_x16,
				4105	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__wasmsimd_x16,
				4106	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__wasmsimd_x16,
				4107	.element_tile = 16,
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	4108	};
Marat Dukhan	d816f62	2020-07-15 10:14:39 -0700	[diff] [blame]	4109	if (is_wasm_x86) {
				4110	xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan	9531e9f	2020-07-24 15:25:02 -0700	[diff] [blame]	4111	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__wasmsimd_x86_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	4112	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	d816f62	2020-07-15 10:14:39 -0700	[diff] [blame]	4113	.channel_tile = 4,
				4114	.row_tile = 2,
				4115	};
				4116	} else {
				4117	xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan	9531e9f	2020-07-24 15:25:02 -0700	[diff] [blame]	4118	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__wasmsimd_arm_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	4119	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	d816f62	2020-07-15 10:14:39 -0700	[diff] [blame]	4120	.channel_tile = 4,
				4121	.row_tile = 2,
				4122	};
				4123	}
Erich Elsen	6e80fdc	2020-06-09 15:35:37 -0700	[diff] [blame]	4124	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	cfbed0a	2020-12-08 10:01:51 -0800	[diff] [blame]	4125	init_flags \|= XNN_INIT_FLAG_CHW_OPT;
				4126
Frank Barchard	498cb50	2020-11-16 23:50:04 -0800	[diff] [blame]	4127	if (is_wasm_x86) {
				4128	xnn_params.f32.spmm = (struct spmm_parameters) {
				4129	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_32x1__wasmsimd_x86,
				4130	.mr = 32,
				4131	.nr = 1,
				4132	};
				4133	} else {
				4134	xnn_params.f32.spmm = (struct spmm_parameters) {
				4135	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_32x1__wasmsimd_arm,
				4136	.mr = 32,
				4137	.nr = 1,
				4138	};
				4139	}
Erich Elsen	0a1970e	2020-06-10 09:24:59 -0700	[diff] [blame]	4140	xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
				4141	.ukernel_with_symm_padding =
Frank Barchard	2213606	2020-11-24 18:44:46 -0800	[diff] [blame]	4142	(xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__wasmsimd_2x2,
Erich Elsen	0a1970e	2020-06-10 09:24:59 -0700	[diff] [blame]	4143	.output_channel_tile = 4,
				4144	.output_height_tile = 2,
				4145	.output_width_tile = 2,
				4146	};
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4147	if (is_wasm_x86) {
				4148	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
Frank Barchard	412e2f4	2020-12-11 11:40:50 -0800	[diff] [blame]	4149	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__wasmsimd_x86_loadsplat_2x4,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4150	.output_width_tile = 4,
Frank Barchard	97883b8	2020-11-23 13:01:03 -0800	[diff] [blame]	4151	.output_height_tile = 2,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4152	};
				4153	xnn_params.f32.dwconv2d_chw_3x3s2 = (struct dwconv2d_chw_parameters) {
Frank Barchard	412e2f4	2020-12-11 11:40:50 -0800	[diff] [blame]	4154	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__wasmsimd_x86_splat_1x4_acc2,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4155	.output_width_tile = 4,
				4156	.output_height_tile = 1,
				4157	};
				4158	xnn_params.f32.dwconv2d_chw_5x5 = (struct dwconv2d_chw_parameters) {
Frank Barchard	412e2f4	2020-12-11 11:40:50 -0800	[diff] [blame]	4159	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5p2__wasmsimd_x86_splat_3x4,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4160	.output_width_tile = 4,
				4161	.output_height_tile = 3,
				4162	};
				4163	xnn_params.f32.dwconv2d_chw_5x5s2 = (struct dwconv2d_chw_parameters) {
Frank Barchard	412e2f4	2020-12-11 11:40:50 -0800	[diff] [blame]	4164	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__wasmsimd_x86_splat_1x4_acc2,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4165	.output_width_tile = 4,
				4166	.output_height_tile = 1,
				4167	};
				4168	} else {
				4169	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
Frank Barchard	412e2f4	2020-12-11 11:40:50 -0800	[diff] [blame]	4170	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__wasmsimd_arm_loadsplat_2x4,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4171	.output_width_tile = 4,
Frank Barchard	97883b8	2020-11-23 13:01:03 -0800	[diff] [blame]	4172	.output_height_tile = 2,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4173	};
				4174	xnn_params.f32.dwconv2d_chw_3x3s2 = (struct dwconv2d_chw_parameters) {
Frank Barchard	412e2f4	2020-12-11 11:40:50 -0800	[diff] [blame]	4175	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__wasmsimd_arm_splat_1x4_acc4,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4176	.output_width_tile = 4,
				4177	.output_height_tile = 1,
				4178	};
				4179	xnn_params.f32.dwconv2d_chw_5x5 = (struct dwconv2d_chw_parameters) {
Frank Barchard	412e2f4	2020-12-11 11:40:50 -0800	[diff] [blame]	4180	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5p2__wasmsimd_arm_splat_3x4,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4181	.output_width_tile = 4,
				4182	.output_height_tile = 3,
				4183	};
				4184	xnn_params.f32.dwconv2d_chw_5x5s2 = (struct dwconv2d_chw_parameters) {
Frank Barchard	412e2f4	2020-12-11 11:40:50 -0800	[diff] [blame]	4185	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__wasmsimd_arm_splat_1x4_acc2,
Frank Barchard	db5c32d	2020-11-16 23:58:42 -0800	[diff] [blame]	4186	.output_width_tile = 4,
				4187	.output_height_tile = 1,
				4188	};
				4189	}
Marat Dukhan	c5045bf	2020-07-27 18:16:35 -0700	[diff] [blame]	4190	if (is_wasm_x86) {
				4191	xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
				4192	.ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__wasmsimd_x86_x4,
				4193	.channel_tile = 4,
				4194	};
				4195	} else {
				4196	xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
				4197	.ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__wasmsimd_arm_x4,
				4198	.channel_tile = 4,
				4199	};
				4200	}
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	4201	xnn_params.f32.ibilinear_chw = (struct ibilinear_chw_parameters) {
				4202	.ukernel = (xnn_ibilinear_chw_ukernel_function) xnn_f32_ibilinear_chw_ukernel__wasmsimd_p8,
				4203	.channel_tile = 1,
Artsiom Ablavatski	b3ffd58	2021-03-31 13:00:08 -0700	[diff] [blame]	4204	.pixel_tile = 8,
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	4205	};
Erich Elsen	6e80fdc	2020-06-09 15:35:37 -0700	[diff] [blame]	4206	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4207	#endif // XNN_NO_F32_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4208
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	4209	/************************* VCVT micro-kernels *************************/
				4210	#ifndef XNN_NO_VCVT_OPERATORS
				4211	init_flags \|= XNN_INIT_FLAG_VCVT;
				4212
				4213	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__wasmsimd_int16_x16;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	4214	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__wasmsimd_x24;
Marat Dukhan	d52d20b	2021-12-05 09:50:25 -0800	[diff] [blame]	4215	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
				4216	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qs8_vcvt_ukernel__wasmsimd_magic_x32,
				4217	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_wasmsimd_magic_params,
				4218	.element_tile = 32,
				4219	};
				4220	xnn_params.vcvt.f32_to_qu8 = (struct vunary_parameters) {
				4221	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__wasmsimd_magic_x32,
				4222	.init.f32_qu8_cvt = xnn_init_f32_qu8_cvt_wasmsimd_magic_params,
				4223	.element_tile = 32,
				4224	};
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	4225	#endif // XNN_NO_VCVT_OPERATORS
				4226
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4227	/************************** X32 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4228	#ifndef XNN_NO_X32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	4229	init_flags \|= XNN_INIT_FLAG_X32;
				4230
Marat Dukhan	9d4bfa2	2020-07-16 19:07:04 -0700	[diff] [blame]	4231	xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__wasmsimd;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4232	xnn_params.x32.zip = (struct zip_parameters) {
Marat Dukhan	e3b7876	2020-07-16 20:02:58 -0700	[diff] [blame]	4233	.x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__wasmsimd,
				4234	.x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__wasmsimd,
				4235	.x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__wasmsimd,
				4236	.xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__wasmsimd,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4237	};
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	4238	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	ad71b9a	2020-11-20 00:01:51 -0800	[diff] [blame]	4239	xnn_params.x32.depthtospace2d_chw2hwc = (struct depthtospace2d_chw2hwc_parameters) {
				4240	.ukernel = (xnn_depthtospace2d_chw2hwc_ukernel_function) xnn_x32_depthtospace2d_chw2hwc_ukernel__scalar,
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	4241	.channel_tile = 1,
				4242	.pixel_tile = 1,
				4243	};
				4244	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4245	#endif // XNN_NO_X32_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4246
Marat Dukhan	048931b	2020-11-24 20:53:54 -0800	[diff] [blame]	4247	/************************** XX micro-kernels **************************/
				4248	#ifndef XNN_NO_XX_OPERATORS
				4249	init_flags \|= XNN_INIT_FLAG_XX;
				4250
				4251	xnn_params.xx.copy = (xnn_univector_ukernel_function) xnn_xx_copy_ukernel__memcpy;
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	4252	xnn_params.xx.fill = (struct fill_parameters) {
				4253	.ukernel = (xnn_fill_ukernel_function) xnn_xx_fill_ukernel__wasmsimd_x64,
				4254	.row_tile = 1,
				4255	};
Marat Dukhan	0461f2d	2021-08-08 12:36:29 -0700	[diff] [blame]	4256	xnn_params.xx.pad = (struct pad_parameters) {
				4257	.ukernel = (xnn_pad_ukernel_function) xnn_xx_pad_ukernel__wasmsimd,
				4258	.row_tile = 1,
				4259	};
Marat Dukhan	048931b	2020-11-24 20:53:54 -0800	[diff] [blame]	4260	#endif
				4261
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	4262	#elif XNN_ARCH_WASM
				4263
Marat Dukhan	898d585	2021-06-30 21:18:34 -0700	[diff] [blame]	4264	/************************** QC8 micro-kernels **************************/
				4265	#ifndef XNN_NO_QC8_OPERATORS
				4266	init_flags \|= XNN_INIT_FLAG_QC8;
				4267
				4268	if (is_wasm_x86) {
				4269	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_2x2__scalar_magic);
				4270	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_2x2__scalar_magic);
				4271	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x2__scalar_magic);
				4272	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x2__scalar_magic);
				4273	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_scalar_magic_params;
				4274	xnn_params.qc8.gemm.mr = 2;
				4275	xnn_params.qc8.gemm.nr = 2;
				4276	} else {
				4277	xnn_params.qc8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_4x4__scalar_magic);
				4278	xnn_params.qc8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_4x4__scalar_magic);
				4279	xnn_params.qc8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qc8_gemm_minmax_fp32_ukernel_1x4__scalar_magic);
				4280	xnn_params.qc8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qc8_igemm_minmax_fp32_ukernel_1x4__scalar_magic);
				4281	xnn_params.qc8.gemm.init.qc8 = xnn_init_qs8_minmax_scalar_magic_params;
				4282	xnn_params.qc8.gemm.mr = 4;
				4283	xnn_params.qc8.gemm.nr = 4;
				4284	}
				4285
				4286	xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up2x9__scalar_magic;
				4287	xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_scalar_magic_params;
				4288	xnn_params.qc8.dwconv[0].channel_tile = 2;
				4289	xnn_params.qc8.dwconv[0].primary_tile = 9;
				4290	xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up2x25__scalar_magic;
				4291	xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_scalar_magic_params;
				4292	xnn_params.qc8.dwconv[1].channel_tile = 2;
				4293	xnn_params.qc8.dwconv[1].primary_tile = 25;
				4294	#endif // XNN_NO_QC8_OPERATORS
				4295
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4296	/************************** QS8 micro-kernels **************************/
				4297	#ifndef XNN_NO_QS8_OPERATORS
				4298	init_flags \|= XNN_INIT_FLAG_QS8;
				4299
				4300	if (is_wasm_x86) {
Marat Dukhan	3d76e55	2021-07-15 18:54:01 -0700	[diff] [blame]	4301	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_2x2__scalar_magic);
				4302	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_2x2__scalar_magic);
				4303	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x2__scalar_magic);
				4304	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x2__scalar_magic);
				4305	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_scalar_magic_params;
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4306	xnn_params.qs8.gemm.mr = 2;
				4307	xnn_params.qs8.gemm.nr = 2;
				4308	} else {
Marat Dukhan	3d76e55	2021-07-15 18:54:01 -0700	[diff] [blame]	4309	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_4x4__scalar_magic);
				4310	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_4x4__scalar_magic);
				4311	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4__scalar_magic);
				4312	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4__scalar_magic);
				4313	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_scalar_magic_params;
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4314	xnn_params.qs8.gemm.mr = 4;
				4315	xnn_params.qs8.gemm.nr = 4;
				4316	}
				4317
Marat Dukhan	3d76e55	2021-07-15 18:54:01 -0700	[diff] [blame]	4318	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up2x9__scalar_magic;
				4319	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_fp32_scalar_magic_params;
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4320	xnn_params.qs8.dwconv[0].channel_tile = 2;
				4321	xnn_params.qs8.dwconv[0].primary_tile = 9;
Marat Dukhan	3d76e55	2021-07-15 18:54:01 -0700	[diff] [blame]	4322	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_fp32_ukernel_up2x25__scalar_magic;
				4323	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_fp32_scalar_magic_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	4324	xnn_params.qs8.dwconv[1].channel_tile = 2;
				4325	xnn_params.qs8.dwconv[1].primary_tile = 25;
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4326
				4327	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				4328	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__scalar_c4,
				4329	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__scalar_c4,
				4330	.mr = 7,
				4331	};
				4332
				4333	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				4334	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__scalar_x4,
				4335	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__scalar_x4,
				4336	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__scalar_x4,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	4337	.init.qs8_addsub = xnn_init_qs8_add_minmax_scalar_params,
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4338	.element_tile = 4,
				4339	};
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	4340	xnn_params.qs8.vmul = (struct vbinary_parameters) {
				4341	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmul_minmax_fp32_ukernel__scalar_x4,
				4342	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__scalar_x4,
				4343	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vmulc_minmax_fp32_ukernel__scalar_x4,
				4344	.init.qs8_mul = xnn_init_qs8_mul_minmax_fp32_scalar_params,
				4345	.element_tile = 4,
				4346	};
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4347	#endif // XNN_NO_QS8_OPERATORS
				4348
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	4349	/************************** QU8 micro-kernels **************************/
				4350	#ifndef XNN_NO_QU8_OPERATORS
				4351	init_flags \|= XNN_INIT_FLAG_QU8;
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	4352
Marat Dukhan	3d76e55	2021-07-15 18:54:01 -0700	[diff] [blame]	4353	if (is_wasm_x86) {
				4354	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_2x2__scalar_magic);
				4355	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_2x2__scalar_magic);
				4356	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x2__scalar_magic);
				4357	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x2__scalar_magic);
				4358	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_scalar_magic_params;
				4359	xnn_params.qu8.gemm.mr = 2;
				4360	xnn_params.qu8.gemm.nr = 2;
				4361	} else {
				4362	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_4x4__scalar_magic);
				4363	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_4x4__scalar_magic);
				4364	xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4__scalar_magic);
				4365	xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4__scalar_magic);
				4366	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_scalar_magic_params;
				4367	xnn_params.qu8.gemm.mr = 4;
				4368	xnn_params.qu8.gemm.nr = 4;
				4369	}
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4370
Marat Dukhan	3d76e55	2021-07-15 18:54:01 -0700	[diff] [blame]	4371	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up2x9__scalar_magic;
				4372	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_fp32_scalar_magic_params;
				4373	xnn_params.qu8.dwconv[0].channel_tile = 2;
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	4374	xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhan	3d76e55	2021-07-15 18:54:01 -0700	[diff] [blame]	4375	xnn_params.qu8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_fp32_ukernel_up2x25__scalar_magic;
				4376	xnn_params.qu8.dwconv[1].init.qu8 = xnn_init_qu8_conv_minmax_fp32_scalar_magic_params;
				4377	xnn_params.qu8.dwconv[1].channel_tile = 2;
				4378	xnn_params.qu8.dwconv[1].primary_tile = 25;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4379
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	4380	xnn_params.qu8.avgpool = (struct avgpool_parameters) {
				4381	.up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__scalar_c1,
				4382	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4383	.mr = 9,
				4384	.qr = 8,
				4385	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	4386	xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
				4387	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__scalar_c1,
				4388	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4389	.mr = 7,
				4390	};
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	4391
				4392	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				4393	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__scalar_x4,
				4394	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__scalar_x4,
				4395	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__scalar_x4,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	4396	.init.qu8_addsub = xnn_init_qu8_add_minmax_scalar_params,
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	4397	.element_tile = 4,
				4398	};
Marat Dukhan	0853b8a	2021-08-03 01:01:53 -0700	[diff] [blame]	4399	xnn_params.qu8.vmul = (struct vbinary_parameters) {
				4400	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmul_minmax_fp32_ukernel__scalar_x4,
				4401	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__scalar_x4,
				4402	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vmulc_minmax_fp32_ukernel__scalar_x4,
				4403	.init.qu8_mul = xnn_init_qu8_mul_minmax_fp32_scalar_params,
				4404	.element_tile = 4,
				4405	};
Marat Dukhan	08b7a97	2020-07-14 18:17:29 -0700	[diff] [blame]	4406	#endif // XNN_NO_QU8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4407
Marat Dukhan	dc5c148	2021-08-16 09:03:15 -0700	[diff] [blame]	4408	/************************** S8 micro-kernels **************************/
				4409	#ifndef XNN_NO_S8_OPERATORS
				4410	init_flags \|= XNN_INIT_FLAG_S8;
				4411
Marat Dukhan	61c0c9e	2021-08-16 23:16:14 -0700	[diff] [blame]	4412	xnn_params.s8.clamp = (struct vunary_parameters) {
				4413	.ukernel = (xnn_univector_ukernel_function) xnn_s8_vclamp_ukernel__scalar_x4,
				4414	.init.s8_minmax = xnn_init_s8_minmax_scalar_params,
				4415	.element_tile = 4,
				4416	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	4417	xnn_params.s8.ibilinear = (struct ibilinear_parameters) {
				4418	.ukernel = (xnn_ibilinear_ukernel_function) xnn_s8_ibilinear_ukernel__scalar_c1,
				4419	.pixel_tile = 1,
				4420	.channel_tile = 1,
				4421	};
Marat Dukhan	dc5c148	2021-08-16 09:03:15 -0700	[diff] [blame]	4422	xnn_params.s8.maxpool = (struct maxpool_parameters) {
				4423	.ukernel = (xnn_maxpool_ukernel_function) xnn_s8_maxpool_minmax_ukernel_9p8x__scalar_c1,
				4424	.init.s8 = xnn_init_s8_minmax_scalar_params,
				4425	.mr = 9,
				4426	.qr = 8,
				4427	};
				4428	#endif // XNN_NO_S8_OPERATORS
				4429
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4430	/************************** U8 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4431	#ifndef XNN_NO_U8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	4432	init_flags \|= XNN_INIT_FLAG_U8;
				4433
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	4434	xnn_params.u8.clamp = (struct vunary_parameters) {
				4435	.ukernel = (xnn_univector_ukernel_function) xnn_u8_vclamp_ukernel__scalar_x4,
				4436	.init.u8_minmax = xnn_init_u8_minmax_scalar_params,
				4437	.element_tile = 4,
				4438	};
Marat Dukhan	24abe6b	2021-11-24 15:28:57 -0800	[diff] [blame]	4439	xnn_params.u8.ibilinear = (struct ibilinear_parameters) {
				4440	.ukernel = (xnn_ibilinear_ukernel_function) xnn_u8_ibilinear_ukernel__scalar_c1,
				4441	.pixel_tile = 1,
				4442	.channel_tile = 1,
				4443	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4444	xnn_params.u8.maxpool = (struct maxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	4445	.ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	4446	.init.u8 = xnn_init_u8_minmax_scalar_params,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4447	.mr = 9,
				4448	.qr = 8,
				4449	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4450	xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
				4451	xnn_params.u8.rmax = xnn_u8_rmax_ukernel__scalar;
				4452	#endif // XNN_NO_U8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4453
				4454	/************************** X8 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4455	#ifndef XNN_NO_X8_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	4456	init_flags \|= XNN_INIT_FLAG_X8;
				4457
Marat Dukhan	d67539d	2021-09-08 23:06:03 -0700	[diff] [blame]	4458	xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar_x4;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4459	xnn_params.x8.zip = (struct zip_parameters) {
				4460	.x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__scalar,
				4461	.x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__scalar,
				4462	.x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__scalar,
				4463	.xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__scalar,
				4464	};
				4465	#endif // XNN_NO_X8_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4466
				4467	/************************** F32 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4468	#ifndef XNN_NO_F32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	4469	init_flags \|= XNN_INIT_FLAG_F32;
				4470
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4471	if (is_wasm_x86) {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4472	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_2x4__scalar);
				4473	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_2x4__scalar);
				4474	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x4__wasm);
				4475	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x4__wasm);
Marat Dukhan	467f636	2020-05-22 23:21:55 -0700	[diff] [blame]	4476	xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_2x4__scalar);
				4477	xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_2x4__scalar);
				4478	xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x4__wasm);
				4479	xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x4__wasm);
Marat Dukhan	869c62d	2020-04-09 17:17:55 -0700	[diff] [blame]	4480	xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_2x4__scalar);
				4481	xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_2x4__scalar);
				4482	xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x4__wasm);
				4483	xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x4__wasm);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4484	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4485	xnn_params.f32.gemm.mr = 2;
				4486	xnn_params.f32.gemm.nr = 4;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4487	} else {
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4488	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x4__wasm);
				4489	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x4__wasm);
				4490	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x4__wasm);
				4491	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x4__wasm);
Marat Dukhan	467f636	2020-05-22 23:21:55 -0700	[diff] [blame]	4492	xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_4x4__wasm);
				4493	xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_4x4__wasm);
				4494	xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x4__wasm);
				4495	xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x4__wasm);
Marat Dukhan	869c62d	2020-04-09 17:17:55 -0700	[diff] [blame]	4496	xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x4__wasm);
				4497	xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x4__wasm);
				4498	xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x4__wasm);
				4499	xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x4__wasm);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4500	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4501	xnn_params.f32.gemm.mr = 4;
				4502	xnn_params.f32.gemm.nr = 4;
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4503	}
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4504	xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__wasm);
				4505	xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__wasm),
Marat Dukhan	869c62d	2020-04-09 17:17:55 -0700	[diff] [blame]	4506	xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2__wasm);
				4507	xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2__wasm),
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4508	xnn_params.f32.gemm2.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4509	xnn_params.f32.gemm2.mr = 4;
				4510	xnn_params.f32.gemm2.nr = 2;
				4511
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4512	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x3__wasm_acc2;
				4513	xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x3__wasm_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4514	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4515	xnn_params.f32.dwconv[0].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4516	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4517
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4518	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x4__wasm_acc2;
				4519	xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x4__wasm_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4520	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4521	xnn_params.f32.dwconv[1].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4522	xnn_params.f32.dwconv[1].primary_tile = 4;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4523
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4524	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x9__wasm_acc2;
				4525	xnn_params.f32.dwconv[2].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x9__wasm_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4526	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4527	xnn_params.f32.dwconv[2].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4528	xnn_params.f32.dwconv[2].primary_tile = 9;
				4529
				4530	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x25__wasm_acc2;
				4531	xnn_params.f32.dwconv[3].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x25__wasm_acc2;
				4532	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_scalar_params;
				4533	xnn_params.f32.dwconv[3].channel_tile = 1;
				4534	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	aefaef3	2020-04-09 07:09:34 -0700	[diff] [blame]	4535
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4536	xnn_params.f32.avgpool = (struct avgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	4537	.up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__wasm_c1,
				4538	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__wasm_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4539	.mr = 9,
				4540	.qr = 8,
				4541	};
				4542	xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	4543	.up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__wasm_c1,
				4544	.mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__wasm_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4545	.mr = 9,
				4546	.qr = 8,
				4547	};
				4548	xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	4549	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__wasm_c1,
				4550	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__wasm_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4551	.mr = 7,
				4552	};
				4553	xnn_params.f32.maxpool = (struct maxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	4554	.ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__wasm_c1,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	4555	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4556	.mr = 9,
				4557	.qr = 8,
				4558	};
				4559	xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	4560	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__scalar_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4561	.mr = 4,
				4562	};
				4563	xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	4564	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__scalar_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4565	.mr = 9,
				4566	};
				4567	xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhan	9993660	2020-04-11 16:47:01 -0700	[diff] [blame]	4568	.mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__scalar_c1,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4569	.mr = 9,
				4570	.qr = 8,
				4571	};
Marat Dukhan	660fd19	2020-03-10 04:55:30 -0700	[diff] [blame]	4572	xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
				4573	.ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__scalar_c2,
Marat Dukhan	6972249	2019-11-11 19:55:50 -0800	[diff] [blame]	4574	.pixel_tile = 1,
				4575	.channel_tile = 2,
				4576	};
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	4577	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__scalar_x4;
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	4578	xnn_params.f32.clamp = (struct vunary_parameters) {
				4579	.ukernel = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__wasm_x4,
				4580	.init.f32_minmax = xnn_init_f32_minmax_scalar_params,
				4581	.element_tile = 4,
				4582	};
Marat Dukhan	c303fe6	2020-06-26 10:09:25 -0700	[diff] [blame]	4583	if (is_wasm_x86) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	4584	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__scalar_x4;
Marat Dukhan	c303fe6	2020-06-26 10:09:25 -0700	[diff] [blame]	4585	} else {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	4586	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__wasm_x4;
Marat Dukhan	c303fe6	2020-06-26 10:09:25 -0700	[diff] [blame]	4587	}
Marat Dukhan	b6bd4bc	2020-12-01 17:01:40 -0800	[diff] [blame]	4588	if (is_wasm_x86) {
				4589	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__scalar_rr2_lut16_p3_x2;
				4590	} else {
				4591	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__wasm_rr2_p6_x6;
				4592	}
Marat Dukhan	2881333	2020-06-10 18:05:38 -0700	[diff] [blame]	4593	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__scalar_x4;
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	4594	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__scalar_x4;
Frank Barchard	62c5e23	2020-07-21 17:42:19 -0700	[diff] [blame]	4595	if (is_wasm_x86) {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	4596	xnn_params.f32.relu = (xnn_univector_ukernel_function) xnn_f32_vrelu_ukernel__scalar_x8;
Frank Barchard	62c5e23	2020-07-21 17:42:19 -0700	[diff] [blame]	4597	} else {
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	4598	xnn_params.f32.relu = (xnn_univector_ukernel_function) xnn_f32_vrelu_ukernel__wasm_x8;
Frank Barchard	62c5e23	2020-07-21 17:42:19 -0700	[diff] [blame]	4599	}
Marat Dukhan	64e5251	2020-06-09 13:41:16 -0700	[diff] [blame]	4600	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__scalar_libm_x4;
				4601	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__scalar_libm_x4;
				4602	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__scalar_libm_x4;
				4603	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__scalar_libm_x4;
Marat Dukhan	6674d69	2021-05-05 22:27:00 -0700	[diff] [blame]	4604	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__scalar_lut64_p2_div_x2;
Marat Dukhan	5020b96	2020-06-08 13:30:10 -0700	[diff] [blame]	4605	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__scalar_x4;
Marat Dukhan	6804bbd	2020-06-30 19:26:11 -0700	[diff] [blame]	4606	xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__scalar_sqrt_x1;
Marat Dukhan	7c1f808	2020-06-25 13:26:20 -0700	[diff] [blame]	4607	if (is_wasm_x86) {
				4608	xnn_params.f32.prelu = (struct prelu_parameters) {
				4609	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__scalar_2x4,
				4610	.row_tile = 2,
				4611	.channel_tile = 4,
				4612	};
				4613	} else {
				4614	xnn_params.f32.prelu = (struct prelu_parameters) {
				4615	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__wasm_2x4,
				4616	.row_tile = 2,
				4617	.channel_tile = 4,
				4618	};
				4619	}
Marat Dukhan	1edc454	2020-01-27 12:40:13 -0800	[diff] [blame]	4620	xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__scalar_p5_x4_acc2;
				4621	xnn_params.f32.rmax = xnn_f32_rmax_ukernel__scalar;
Marat Dukhan	b1a0fc3	2019-12-02 19:32:02 -0800	[diff] [blame]	4622	xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4623	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__wasm_x8,
				4624	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasm_x8,
				4625	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasm_x8,
Marat Dukhan	b1a0fc3	2019-12-02 19:32:02 -0800	[diff] [blame]	4626	.element_tile = 8,
				4627	};
Marat Dukhan	6918050	2019-12-06 15:00:31 -0800	[diff] [blame]	4628	xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchard	b392f8e	2020-10-27 10:46:44 -0700	[diff] [blame]	4629	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__wasm_x8,
				4630	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__wasm_x8,
				4631	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__wasm_x8,
				4632	.element_tile = 8,
Marat Dukhan	6918050	2019-12-06 15:00:31 -0800	[diff] [blame]	4633	};
Marat Dukhan	79e7f84	2019-12-05 14:35:50 -0800	[diff] [blame]	4634	xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4635	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__wasm_x8,
				4636	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasm_x8,
				4637	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasm_x8,
Marat Dukhan	79e7f84	2019-12-05 14:35:50 -0800	[diff] [blame]	4638	.element_tile = 8,
				4639	};
				4640	xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4641	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__wasm_x8,
				4642	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasm_x8,
				4643	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasm_x8,
Marat Dukhan	79e7f84	2019-12-05 14:35:50 -0800	[diff] [blame]	4644	.element_tile = 8,
				4645	};
Marat Dukhan	1e782c4	2019-11-21 17:02:40 -0800	[diff] [blame]	4646	xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4647	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__wasm_x8,
				4648	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasm_x8,
				4649	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasm_x8,
Marat Dukhan	ca2733c	2019-11-15 23:21:17 -0800	[diff] [blame]	4650	.element_tile = 8,
				4651	};
Marat Dukhan	05f3f6d	2019-12-03 15:13:53 -0800	[diff] [blame]	4652	xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4653	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__wasm_x8,
				4654	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__wasm_x8,
				4655	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__wasm_x8,
Marat Dukhan	05f3f6d	2019-12-03 15:13:53 -0800	[diff] [blame]	4656	.element_tile = 8,
				4657	};
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	4658	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard	9c7308f	2020-08-31 17:03:01 -0700	[diff] [blame]	4659	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__scalar_x8,
				4660	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
				4661	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
Marat Dukhan	f739926	2020-06-05 10:58:44 -0700	[diff] [blame]	4662	.element_tile = 8,
				4663	};
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4664	xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan	9531e9f	2020-07-24 15:25:02 -0700	[diff] [blame]	4665	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c1__wasm_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	4666	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	49e6ee9	2019-11-06 15:55:29 -0800	[diff] [blame]	4667	.channel_tile = 1,
				4668	.row_tile = 2,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4669	};
Marat Dukhan	efc47b8	2019-11-18 09:25:38 -0800	[diff] [blame]	4670	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	cfbed0a	2020-12-08 10:01:51 -0800	[diff] [blame]	4671	init_flags \|= XNN_INIT_FLAG_CHW_OPT;
				4672
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4673	xnn_params.f32.spmm = (struct spmm_parameters) {
Marat Dukhan	355ab43	2020-04-09 19:01:52 -0700	[diff] [blame]	4674	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x1__scalar,
Marat Dukhan	bff791e	2019-10-24 11:05:37 -0700	[diff] [blame]	4675	.mr = 8,
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4676	.nr = 1,
				4677	};
Erich Elsen	c6afd9b	2019-10-24 16:10:53 -0700	[diff] [blame]	4678	xnn_params.f32.spmm2 = (struct spmm_parameters) {
Marat Dukhan	355ab43	2020-04-09 19:01:52 -0700	[diff] [blame]	4679	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x2__scalar,
Erich Elsen	c6afd9b	2019-10-24 16:10:53 -0700	[diff] [blame]	4680	.mr = 8,
				4681	.nr = 2,
				4682	};
				4683	xnn_params.f32.spmm4 = (struct spmm_parameters) {
Marat Dukhan	355ab43	2020-04-09 19:01:52 -0700	[diff] [blame]	4684	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x4__scalar,
Erich Elsen	c6afd9b	2019-10-24 16:10:53 -0700	[diff] [blame]	4685	.mr = 8,
				4686	.nr = 4,
				4687	};
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	4688	xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
Marat Dukhan	14fe0b2	2019-10-23 21:20:07 -0700	[diff] [blame]	4689	.ukernel_with_symm_padding =
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	4690	(xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__scalar_1x1,
Marat Dukhan	14fe0b2	2019-10-23 21:20:07 -0700	[diff] [blame]	4691	.output_channel_tile = 4,
				4692	.output_height_tile = 1,
				4693	.output_width_tile = 1,
				4694	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	4695	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
Marat Dukhan	91249d2	2020-10-24 12:02:51 -0700	[diff] [blame]	4696	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__scalar_2x1_acc2,
Marat Dukhan	14fe0b2	2019-10-23 21:20:07 -0700	[diff] [blame]	4697	.output_width_tile = 1,
Marat Dukhan	91249d2	2020-10-24 12:02:51 -0700	[diff] [blame]	4698	.output_height_tile = 2,
Marat Dukhan	14fe0b2	2019-10-23 21:20:07 -0700	[diff] [blame]	4699	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	4700	xnn_params.f32.dwconv2d_chw_3x3s2 = (struct dwconv2d_chw_parameters) {
Marat Dukhan	cf5b3c3	2020-10-25 19:21:10 -0700	[diff] [blame]	4701	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__scalar_1x1_acc2,
Marat Dukhan	14fe0b2	2019-10-23 21:20:07 -0700	[diff] [blame]	4702	.output_width_tile = 1,
				4703	.output_height_tile = 1,
				4704	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	4705	xnn_params.f32.dwconv2d_chw_5x5 = (struct dwconv2d_chw_parameters) {
				4706	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5p2__scalar_1x1_acc5,
Marat Dukhan	a99918a	2019-11-15 14:40:12 -0800	[diff] [blame]	4707	.output_width_tile = 1,
				4708	.output_height_tile = 1,
				4709	};
Marat Dukhan	bf715f9	2020-10-23 20:17:00 -0700	[diff] [blame]	4710	xnn_params.f32.dwconv2d_chw_5x5s2 = (struct dwconv2d_chw_parameters) {
				4711	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__scalar_1x1_acc5,
Marat Dukhan	a99918a	2019-11-15 14:40:12 -0800	[diff] [blame]	4712	.output_width_tile = 1,
				4713	.output_height_tile = 1,
				4714	};
Marat Dukhan	1f29b80	2020-05-15 23:46:39 -0700	[diff] [blame]	4715	xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
				4716	.ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__scalar_x1,
Marat Dukhan	14fe0b2	2019-10-23 21:20:07 -0700	[diff] [blame]	4717	.channel_tile = 1,
				4718	};
Artsiom Ablavatski	9791810	2020-10-27 15:52:59 -0700	[diff] [blame]	4719	xnn_params.f32.ibilinear_chw = (struct ibilinear_chw_parameters) {
				4720	.ukernel = (xnn_ibilinear_chw_ukernel_function) xnn_f32_ibilinear_chw_ukernel__scalar_p4,
				4721	.channel_tile = 1,
				4722	.pixel_tile = 4,
				4723	};
Marat Dukhan	efc47b8	2019-11-18 09:25:38 -0800	[diff] [blame]	4724	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4725	#endif // XNN_NO_F32_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4726
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	4727	/************************* VCVT micro-kernels *************************/
				4728	#ifndef XNN_NO_VCVT_OPERATORS
				4729	init_flags \|= XNN_INIT_FLAG_VCVT;
				4730
				4731	xnn_params.vcvt.f16_to_f32 = (xnn_univector_ukernel_function) xnn_f16_f32_vcvt_ukernel__scalar_float_x1;
Marat Dukhan	a0c6168	2021-11-10 19:23:41 -0800	[diff] [blame]	4732	xnn_params.vcvt.f32_to_f16 = (xnn_univector_ukernel_function) xnn_f32_f16_vcvt_ukernel__scalar_bitcast_x4;
Marat Dukhan	430b173	2021-12-04 02:53:12 -0800	[diff] [blame]	4733	if (is_wasm_x86) {
				4734	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
				4735	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__scalar_magic_iminmax_x1,
				4736	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_scalar_magic_iminmax_params,
				4737	.element_tile = 1,
				4738	};
				4739	xnn_params.vcvt.f32_to_qu8 = (struct vunary_parameters) {
				4740	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__scalar_magic_iminmax_x1,
				4741	.init.f32_qu8_cvt = xnn_init_f32_qu8_cvt_scalar_magic_iminmax_params,
				4742	.element_tile = 1,
				4743	};
				4744	} else {
				4745	xnn_params.vcvt.f32_to_qs8 = (struct vunary_parameters) {
				4746	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__wasm_magic_fminmax_x4,
				4747	.init.f32_qs8_cvt = xnn_init_f32_qs8_cvt_scalar_magic_fminmax_params,
				4748	.element_tile = 4,
				4749	};
				4750	xnn_params.vcvt.f32_to_qu8 = (struct vunary_parameters) {
				4751	.ukernel = (xnn_univector_ukernel_function) xnn_f32_qu8_vcvt_ukernel__wasm_magic_fminmax_x4,
				4752	.init.f32_qu8_cvt = xnn_init_f32_qu8_cvt_scalar_magic_fminmax_params,
				4753	.element_tile = 4,
				4754	};
				4755	}
Marat Dukhan	af2ba00	2021-10-24 14:21:41 -0700	[diff] [blame]	4756	#endif // XNN_NO_VCVT_OPERATORS
				4757
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4758	/************************** X32 micro-kernels **************************/
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4759	#ifndef XNN_NO_X32_OPERATORS
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	4760	init_flags \|= XNN_INIT_FLAG_X32;
				4761
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4762	xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__scalar;
				4763	xnn_params.x32.zip = (struct zip_parameters) {
				4764	.x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__scalar,
				4765	.x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__scalar,
				4766	.x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__scalar,
				4767	.xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__scalar,
				4768	};
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	4769	#ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan	ad71b9a	2020-11-20 00:01:51 -0800	[diff] [blame]	4770	xnn_params.x32.depthtospace2d_chw2hwc = (struct depthtospace2d_chw2hwc_parameters) {
				4771	.ukernel = (xnn_depthtospace2d_chw2hwc_ukernel_function) xnn_x32_depthtospace2d_chw2hwc_ukernel__scalar,
Artsiom Ablavatski	bbe8506	2020-11-05 14:07:37 -0800	[diff] [blame]	4772	.channel_tile = 1,
				4773	.pixel_tile = 1,
				4774	};
				4775	#endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan	8fe54e4	2019-10-10 14:12:59 -0700	[diff] [blame]	4776	#endif // XNN_NO_X32_OPERATORS
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	4777
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	4778	/************************** XX micro-kernels **************************/
				4779	#ifndef XNN_NO_XX_OPERATORS
				4780	init_flags \|= XNN_INIT_FLAG_XX;
				4781
				4782	xnn_params.xx.copy = (xnn_univector_ukernel_function) xnn_xx_copy_ukernel__memcpy;
				4783	xnn_params.xx.fill = (struct fill_parameters) {
				4784	.ukernel = (xnn_fill_ukernel_function) xnn_xx_fill_ukernel__scalar_x16,
				4785	.row_tile = 1,
				4786	};
Marat Dukhan	0461f2d	2021-08-08 12:36:29 -0700	[diff] [blame]	4787	xnn_params.xx.pad = (struct pad_parameters) {
				4788	.ukernel = (xnn_pad_ukernel_function) xnn_xx_pad_ukernel__scalar,
				4789	.row_tile = 1,
				4790	};
Marat Dukhan	933051b	2021-08-07 16:26:15 -0700	[diff] [blame]	4791	#endif
				4792
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4793	#elif XNN_ARCH_RISCV
				4794
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4795	/************************** QS8 micro-kernels **************************/
				4796	#ifndef XNN_NO_QS8_OPERATORS
				4797	init_flags \|= XNN_INIT_FLAG_QS8;
				4798
Frank Barchard	22f9a9f	2021-07-21 11:35:27 -0700	[diff] [blame]	4799	xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_3x4__scalar);
				4800	xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_3x4__scalar);
				4801	xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_rndnu_ukernel_1x4__scalar);
				4802	xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_rndnu_ukernel_1x4__scalar);
				4803	xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_rndnu_scalar_params;
Marat Dukhan	15a35c0	2021-05-12 11:40:03 -0700	[diff] [blame]	4804	xnn_params.qs8.gemm.mr = 3;
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4805	xnn_params.qs8.gemm.nr = 4;
				4806
Marat Dukhan	b07c26a	2021-05-24 19:44:51 -0700	[diff] [blame]	4807	xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up2x9__scalar;
Marat Dukhan	9b474cf	2021-05-25 16:37:48 -0700	[diff] [blame]	4808	xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4809	xnn_params.qs8.dwconv[0].channel_tile = 2;
				4810	xnn_params.qs8.dwconv[0].primary_tile = 9;
Marat Dukhan	b07c26a	2021-05-24 19:44:51 -0700	[diff] [blame]	4811	xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_gemmlowp_ukernel_up2x25__scalar;
Marat Dukhan	9b474cf	2021-05-25 16:37:48 -0700	[diff] [blame]	4812	xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_scalar_params;
Marat Dukhan	4ed1488	2021-05-12 17:50:40 -0700	[diff] [blame]	4813	xnn_params.qs8.dwconv[1].channel_tile = 2;
				4814	xnn_params.qs8.dwconv[1].primary_tile = 25;
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4815
				4816	xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
				4817	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__scalar_c1,
				4818	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
				4819	.mr = 7,
				4820	};
				4821
				4822	xnn_params.qs8.vadd = (struct vbinary_parameters) {
				4823	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__scalar_x4,
				4824	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__scalar_x4,
				4825	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__scalar_x4,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	4826	.init.qs8_addsub = xnn_init_qs8_add_minmax_scalar_params,
Marat Dukhan	803c1f8	2021-05-12 00:13:37 -0700	[diff] [blame]	4827	.element_tile = 4,
				4828	};
				4829	#endif // XNN_NO_QS8_OPERATORS
				4830
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4831	/************************** QU8 micro-kernels **************************/
				4832	#ifndef XNN_NO_QU8_OPERATORS
				4833	init_flags \|= XNN_INIT_FLAG_QU8;
				4834
Marat Dukhan	c2e8f66	2021-07-01 17:06:34 -0700	[diff] [blame]	4835	xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_gemmlowp_ukernel_2x2__scalar);
				4836	xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_gemmlowp_ukernel_2x2__scalar);
Marat Dukhan	c698c11	2021-07-01 18:52:10 -0700	[diff] [blame]	4837	xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_gemmlowp_scalar_params;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4838	xnn_params.qu8.gemm.mr = 2;
				4839	xnn_params.qu8.gemm.nr = 2;
				4840
Marat Dukhan	c2e8f66	2021-07-01 17:06:34 -0700	[diff] [blame]	4841	xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_gemmlowp_ukernel_up1x9__scalar;
Marat Dukhan	c698c11	2021-07-01 18:52:10 -0700	[diff] [blame]	4842	xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_gemmlowp_scalar_params;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4843	xnn_params.qu8.dwconv[0].channel_tile = 1;
				4844	xnn_params.qu8.dwconv[0].primary_tile = 9;
				4845
				4846	xnn_params.qu8.avgpool = (struct avgpool_parameters) {
				4847	.up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__scalar_c1,
				4848	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__scalar_c1,
				4849	.mr = 9,
				4850	.qr = 8,
				4851	};
				4852	xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
				4853	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__scalar_c1,
				4854	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
				4855	.mr = 7,
				4856	};
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	4857
				4858	xnn_params.qu8.vadd = (struct vbinary_parameters) {
				4859	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vadd_minmax_ukernel__scalar_x4,
				4860	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__scalar_x4,
				4861	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qu8_vaddc_minmax_ukernel__scalar_x4,
Marat Dukhan	6428725	2021-09-07 16:20:03 -0700	[diff] [blame]	4862	.init.qu8_addsub = xnn_init_qu8_add_minmax_scalar_params,
Marat Dukhan	db007cd	2021-07-20 23:42:39 -0700	[diff] [blame]	4863	.element_tile = 4,
				4864	};
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4865	#endif // XNN_NO_QU8_OPERATORS
				4866
				4867	/************************** U8 micro-kernels **************************/
				4868	#ifndef XNN_NO_U8_OPERATORS
				4869	init_flags \|= XNN_INIT_FLAG_U8;
				4870
Marat Dukhan	9491279	2021-08-16 21:40:30 -0700	[diff] [blame]	4871	xnn_params.u8.clamp = (struct vunary_parameters) {
				4872	.ukernel = (xnn_univector_ukernel_function) xnn_u8_vclamp_ukernel__scalar_x4,
				4873	.init.u8_minmax = xnn_init_u8_minmax_scalar_params,
				4874	.element_tile = 4,
				4875	};
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4876	xnn_params.u8.maxpool = (struct maxpool_parameters) {
				4877	.ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	4878	.init.u8 = xnn_init_u8_minmax_scalar_params,
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4879	.mr = 9,
				4880	.qr = 8,
				4881	};
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4882	xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
				4883	xnn_params.u8.rmax = xnn_u8_rmax_ukernel__scalar;
				4884	#endif // XNN_NO_U8_OPERATORS
				4885
				4886	/************************** X8 micro-kernels **************************/
				4887	#ifndef XNN_NO_X8_OPERATORS
				4888	init_flags \|= XNN_INIT_FLAG_X8;
				4889
Marat Dukhan	d67539d	2021-09-08 23:06:03 -0700	[diff] [blame]	4890	xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar_x4;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4891	xnn_params.x8.zip = (struct zip_parameters) {
				4892	.x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__scalar,
				4893	.x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__scalar,
				4894	.x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__scalar,
				4895	.xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__scalar,
				4896	};
				4897	#endif // XNN_NO_X8_OPERATORS
				4898
				4899	/************************** F32 micro-kernels **************************/
				4900	#ifndef XNN_NO_F32_OPERATORS
				4901	init_flags \|= XNN_INIT_FLAG_F32;
				4902
				4903	xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x4__scalar);
				4904	xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x4__scalar);
				4905	xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x4__scalar);
				4906	xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x4__scalar);
				4907	xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_4x4__scalar);
				4908	xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_4x4__scalar);
				4909	xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x4__scalar);
				4910	xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x4__scalar);
				4911	xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x4__scalar);
				4912	xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x4__scalar);
				4913	xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x4__scalar);
				4914	xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x4__scalar);
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4915	xnn_params.f32.gemm.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4916	xnn_params.f32.gemm.mr = 4;
				4917	xnn_params.f32.gemm.nr = 4;
				4918
				4919	xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__scalar);
				4920	xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__scalar),
				4921	xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2__scalar);
				4922	xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2__scalar),
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4923	xnn_params.f32.gemm2.init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4924	xnn_params.f32.gemm2.mr = 4;
				4925	xnn_params.f32.gemm2.nr = 2;
				4926
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4927	xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x3__scalar_acc2;
				4928	xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x3__scalar_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4929	xnn_params.f32.dwconv[0].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4930	xnn_params.f32.dwconv[0].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4931	xnn_params.f32.dwconv[0].primary_tile = 3;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4932
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4933	xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x4__scalar_acc2;
				4934	xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x4__scalar_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4935	xnn_params.f32.dwconv[1].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4936	xnn_params.f32.dwconv[1].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4937	xnn_params.f32.dwconv[1].primary_tile = 4;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4938
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4939	xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x9__scalar_acc2;
				4940	xnn_params.f32.dwconv[2].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x9__scalar_acc2;
Marat Dukhan	104ae5e	2021-05-24 13:41:57 -0700	[diff] [blame]	4941	xnn_params.f32.dwconv[2].init.f32 = xnn_init_f32_minmax_scalar_params;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4942	xnn_params.f32.dwconv[2].channel_tile = 1;
Artsiom Ablavatski	47a74db	2021-11-02 13:40:24 -0700	[diff] [blame]	4943	xnn_params.f32.dwconv[2].primary_tile = 9;
				4944
				4945	xnn_params.f32.dwconv[3].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x25__scalar_acc2;
				4946	xnn_params.f32.dwconv[3].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x25__scalar_acc2;
				4947	xnn_params.f32.dwconv[3].init.f32 = xnn_init_f32_minmax_scalar_params;
				4948	xnn_params.f32.dwconv[3].channel_tile = 1;
				4949	xnn_params.f32.dwconv[3].primary_tile = 25;
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4950
				4951	xnn_params.f32.avgpool = (struct avgpool_parameters) {
				4952	.up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__scalar_c1,
				4953	.mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__scalar_c1,
				4954	.mr = 9,
				4955	.qr = 8,
				4956	};
				4957	xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
				4958	.up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__scalar_c1,
				4959	.mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__scalar_c1,
				4960	.mr = 9,
				4961	.qr = 8,
				4962	};
				4963	xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
				4964	.up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__scalar_c1,
				4965	.mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__scalar_c1,
				4966	.mr = 7,
				4967	};
				4968	xnn_params.f32.maxpool = (struct maxpool_parameters) {
				4969	.ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan	91ae165	2021-08-15 19:19:49 -0700	[diff] [blame]	4970	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	4971	.mr = 9,
				4972	.qr = 8,
				4973	};
				4974	xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
				4975	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__scalar_c1,
				4976	.mr = 4,
				4977	};
				4978	xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
				4979	.up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__scalar_c1,
				4980	.mr = 9,
				4981	};
				4982	xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
				4983	.mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__scalar_c1,
				4984	.mr = 9,
				4985	.qr = 8,
				4986	};
				4987	xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
				4988	.ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__scalar_c2,
				4989	.pixel_tile = 1,
				4990	.channel_tile = 2,
				4991	};
				4992	xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__scalar_x4;
				4993	xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_vclamp_ukernel__scalar_x4;
				4994	xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_vhswish_ukernel__scalar_x4;
				4995	xnn_params.f32.elu = (xnn_univector_ukernel_function) xnn_f32_velu_ukernel__scalar_rr2_lut16_p3_x2;
				4996	xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__scalar_x4;
				4997	xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__scalar_x4;
				4998	xnn_params.f32.relu = (xnn_univector_ukernel_function) xnn_f32_vrelu_ukernel__scalar_x8;
				4999	xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__scalar_libm_x4;
				5000	xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__scalar_libm_x4;
				5001	xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__scalar_libm_x4;
				5002	xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__scalar_libm_x4;
				5003	xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_vsigmoid_ukernel__scalar_lut64_p2_div_x2;
				5004	xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__scalar_x4;
				5005	xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__scalar_sqrt_x1;
				5006	xnn_params.f32.prelu = (struct prelu_parameters) {
				5007	.ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__scalar_2x4,
				5008	.row_tile = 2,
				5009	.channel_tile = 4,
				5010	};
				5011	xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__scalar_p5_x4_acc2;
				5012	xnn_params.f32.rmax = xnn_f32_rmax_ukernel__scalar;
				5013	xnn_params.f32.vadd = (struct vbinary_parameters) {
				5014	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__scalar_x8,
				5015	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__scalar_x8,
				5016	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__scalar_x8,
				5017	.element_tile = 8,
				5018	};
				5019	xnn_params.f32.vdiv = (struct vbinary_parameters) {
				5020	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__scalar_x8,
				5021	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__scalar_x8,
				5022	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__scalar_x8,
				5023	.element_tile = 8,
				5024	};
				5025	xnn_params.f32.vmax = (struct vbinary_parameters) {
				5026	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__scalar_x8,
				5027	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__scalar_x8,
				5028	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__scalar_x8,
				5029	.element_tile = 8,
				5030	};
				5031	xnn_params.f32.vmin = (struct vbinary_parameters) {
				5032	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__scalar_x8,
				5033	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__scalar_x8,
				5034	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__scalar_x8,
				5035	.element_tile = 8,
				5036	};
				5037	xnn_params.f32.vmul = (struct vbinary_parameters) {
				5038	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__scalar_x8,
				5039	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__scalar_x8,
				5040	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__scalar_x8,
				5041	.element_tile = 8,
				5042	};
				5043	xnn_params.f32.vsub = (struct vbinary_parameters) {
				5044	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__scalar_x8,
				5045	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__scalar_x8,
				5046	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__scalar_x8,
				5047	.element_tile = 8,
				5048	};
				5049	xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
				5050	.minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__scalar_x8,
				5051	.minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
				5052	.minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
				5053	.element_tile = 8,
				5054	};
				5055	xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
				5056	.ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c1__scalar_2x,
Marat Dukhan	c58453f	2021-05-24 14:47:38 -0700	[diff] [blame]	5057	.init.f32 = xnn_init_f32_minmax_scalar_params,
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	5058	.channel_tile = 1,
				5059	.row_tile = 2,
				5060	};
				5061	#ifndef XNN_NO_NCHW_OPERATORS
				5062	init_flags \|= XNN_INIT_FLAG_CHW_OPT;
				5063
				5064	xnn_params.f32.spmm = (struct spmm_parameters) {
				5065	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x1__scalar,
				5066	.mr = 8,
				5067	.nr = 1,
				5068	};
				5069	xnn_params.f32.spmm2 = (struct spmm_parameters) {
				5070	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x2__scalar,
				5071	.mr = 8,
				5072	.nr = 2,
				5073	};
				5074	xnn_params.f32.spmm4 = (struct spmm_parameters) {
				5075	.ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x4__scalar,
				5076	.mr = 8,
				5077	.nr = 4,
				5078	};
				5079	xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
				5080	.ukernel_with_symm_padding =
				5081	(xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__scalar_1x1,
				5082	.output_channel_tile = 4,
				5083	.output_height_tile = 1,
				5084	.output_width_tile = 1,
				5085	};
				5086	xnn_params.f32.dwconv2d_chw_3x3 = (struct dwconv2d_chw_parameters) {
				5087	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3p1__scalar_2x1_acc2,
				5088	.output_width_tile = 1,
				5089	.output_height_tile = 2,
				5090	};
				5091	xnn_params.f32.dwconv2d_chw_3x3s2 = (struct dwconv2d_chw_parameters) {
				5092	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__scalar_1x1_acc2,
				5093	.output_width_tile = 1,
				5094	.output_height_tile = 1,
				5095	};
				5096	xnn_params.f32.dwconv2d_chw_5x5 = (struct dwconv2d_chw_parameters) {
				5097	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5p2__scalar_1x1_acc5,
				5098	.output_width_tile = 1,
				5099	.output_height_tile = 1,
				5100	};
				5101	xnn_params.f32.dwconv2d_chw_5x5s2 = (struct dwconv2d_chw_parameters) {
				5102	.ukernel = (xnn_dwconv2d_chw_ukernel_function) xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__scalar_1x1_acc5,
				5103	.output_width_tile = 1,
				5104	.output_height_tile = 1,
				5105	};
				5106	xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
				5107	.ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__scalar_x1,
				5108	.channel_tile = 1,
				5109	};
				5110	xnn_params.f32.ibilinear_chw = (struct ibilinear_chw_parameters) {
				5111	.ukernel = (xnn_ibilinear_chw_ukernel_function) xnn_f32_ibilinear_chw_ukernel__scalar_p4,
				5112	.channel_tile = 1,
				5113	.pixel_tile = 4,
				5114	};
				5115	#endif // XNN_NO_NCHW_OPERATORS
				5116	#endif // XNN_NO_F32_OPERATORS
				5117
				5118	/************************** X32 micro-kernels **************************/
				5119	#ifndef XNN_NO_X32_OPERATORS
				5120	init_flags \|= XNN_INIT_FLAG_X32;
				5121
Marat Dukhan	e3cb19b	2021-05-07 03:38:49 -0700	[diff] [blame]	5122	xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__scalar;
				5123	xnn_params.x32.zip = (struct zip_parameters) {
				5124	.x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__scalar,
				5125	.x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__scalar,
				5126	.x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__scalar,
				5127	.xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__scalar,
				5128	};
				5129	#ifndef XNN_NO_NCHW_OPERATORS
				5130	xnn_params.x32.depthtospace2d_chw2hwc = (struct depthtospace2d_chw2hwc_parameters) {
				5131	.ukernel = (xnn_depthtospace2d_chw2hwc_ukernel_function) xnn_x32_depthtospace2d_chw2hwc_ukernel__scalar,
				5132	.channel_tile = 1,
				5133	.pixel_tile = 1,
				5134	};
				5135	#endif // XNN_NO_NCHW_OPERATORS
				5136	#endif // XNN_NO_X32_OPERATORS
				5137
Marat Dukhan	0461f2d	2021-08-08 12:36:29 -0700	[diff] [blame]	5138	/************************** XX micro-kernels **************************/
				5139	#ifndef XNN_NO_XX_OPERATORS
				5140	init_flags \|= XNN_INIT_FLAG_XX;
				5141
				5142	xnn_params.xx.copy = (xnn_univector_ukernel_function) xnn_xx_copy_ukernel__memcpy;
				5143	xnn_params.xx.fill = (struct fill_parameters) {
				5144	.ukernel = (xnn_fill_ukernel_function) xnn_xx_fill_ukernel__scalar_x16,
				5145	.row_tile = 1,
				5146	};
				5147	xnn_params.xx.pad = (struct pad_parameters) {
				5148	.ukernel = (xnn_pad_ukernel_function) xnn_xx_pad_ukernel__scalar,
				5149	.row_tile = 1,
				5150	};
				5151	#endif
				5152
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	5153	#else
				5154	#error "Unsupported architecture"
				5155	#endif
Marat Dukhan	496389f	2021-04-07 15:47:12 -0700	[diff] [blame]	5156
				5157	memcpy(&xnn_params.allocator, init_allocator, sizeof(struct xnn_allocator));
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	5158	xnn_params.init_flags = init_flags;
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	5159	}
				5160
Marat Dukhan	57133c0	2020-04-13 00:54:59 -0700	[diff] [blame]	5161	#ifdef _WIN32
				5162	static BOOL CALLBACK init_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
				5163	init();
				5164	return TRUE;
				5165	}
				5166	#endif
				5167
Marat Dukhan	04f03be	2019-11-19 12:36:47 -0800	[diff] [blame]	5168	enum xnn_status xnn_initialize(const struct xnn_allocator* allocator) {
Marat Dukhan	d343c22	2019-10-07 09:22:14 -0700	[diff] [blame]	5169	#ifndef __EMSCRIPTEN__
				5170	if (!cpuinfo_initialize()) {
				5171	return xnn_status_out_of_memory;
				5172	}
				5173	#endif
Marat Dukhan	496389f	2021-04-07 15:47:12 -0700	[diff] [blame]	5174	if (allocator == NULL) {
				5175	allocator = &xnn_default_allocator;
				5176	}
				5177	#ifdef _MSC_VER
Marat Dukhan	df94d98	2021-06-01 12:21:33 -0700	[diff] [blame]	5178	_InterlockedCompareExchangePointer((PVOID volatile*) &init_allocator, (PVOID) allocator, NULL);
Marat Dukhan	496389f	2021-04-07 15:47:12 -0700	[diff] [blame]	5179	#else
				5180	__sync_bool_compare_and_swap(&init_allocator, NULL, allocator);
				5181	#endif
Marat Dukhan	57133c0	2020-04-13 00:54:59 -0700	[diff] [blame]	5182	#ifdef _WIN32
				5183	InitOnceExecuteOnce(&init_guard, &init_windows, NULL, NULL);
				5184	#else
				5185	pthread_once(&init_guard, &init);
				5186	#endif
Marat Dukhan	854fb6b	2020-06-19 12:33:44 -0700	[diff] [blame]	5187	if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) != 0) {
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	5188	return xnn_status_success;
				5189	} else {
				5190	return xnn_status_unsupported_hardware;
				5191	}
				5192	}
				5193
				5194	enum xnn_status xnn_deinitialize(void) {
Marat Dukhan	d343c22	2019-10-07 09:22:14 -0700	[diff] [blame]	5195	#ifndef __EMSCRIPTEN__
				5196	cpuinfo_deinitialize();
				5197	#endif
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	5198	return xnn_status_success;
Marat Dukhan	3b9b4bc	2021-08-24 00:23:07 -0700	[diff] [blame]	5199	}