Blame - src/xnnpack/gemm.h - platform/external/XNNPACK

blob: b1d3d7a5dab7ec5dfcc49d2c34aba4de4eddc1e1 [file] [log] [blame]

XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	1	// Copyright (c) Facebook, Inc. and its affiliates.
				2	// All rights reserved.
				3	//
				4	// Copyright 2019 Google LLC
				5	//
				6	// This source code is licensed under the BSD-style license found in the
				7	// LICENSE file in the root directory of this source tree.
				8
				9	#pragma once
				10
				11	#include <stddef.h>
				12	#include <stdint.h>
				13
				14	#include <xnnpack/params.h>
				15	#include <xnnpack/common.h>
				16
				17	#ifdef __cplusplus
				18	extern "C" {
				19	#endif
				20
				21
Marat Dukhan	163a7e6	2020-04-09 04:19:26 -0700	[diff] [blame]	22	#define DECLARE_F32_GEMM_UKERNEL_FUNCTION(fn_name) \
				23	XNN_INTERNAL void fn_name( \
				24	size_t mr, \
				25	size_t nr, \
				26	size_t k, \
				27	const float* a, \
				28	size_t a_stride, \
				29	const float* w, \
				30	float* c, \
				31	size_t cm_stride, \
				32	size_t cn_stride, \
				33	const union xnn_f32_default_params* params);
				34
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	35	#define DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(fn_name) \
				36	XNN_INTERNAL void fn_name( \
				37	size_t mr, \
				38	size_t nr, \
				39	size_t k, \
				40	const float* a, \
				41	size_t a_stride, \
				42	const float* w, \
				43	float* c, \
				44	size_t cm_stride, \
				45	size_t cn_stride, \
Marat Dukhan	eb09a6b	2020-04-08 17:34:32 -0700	[diff] [blame]	46	const union xnn_f32_minmax_params* params);
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	47
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	48	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x2__neon_lane_ld64)
				49	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64)
				50	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__neon_lane_ld64)
				51	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__neon_lane_ld128)
				52	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x8__neon_lane_ld64)
				53	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__neon_lane_ld64)
				54	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__neon_lane_ld128)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	55
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	56	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x2__neonfma_lane_ld64)
				57	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__neonfma_lane_ld64)
				58	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__neonfma_lane_ld64)
				59	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__neonfma_lane_ld128)
				60	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x8__neonfma_lane_ld64)
				61	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__neonfma_lane_ld64)
				62	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__neonfma_lane_ld128)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	63
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	64	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__neon_dup_ld64)
				65	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__neon_dup_ld64)
				66	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__neon_dup_ld128)
				67	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__neon_dup_ld64)
				68	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__neon_dup_ld128)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	69
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	70	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__neonfma_dup_ld64)
				71	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__neonfma_dup_ld64)
				72	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__neonfma_dup_ld128)
				73	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__neonfma_dup_ld64)
				74	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__neonfma_dup_ld128)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	75
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	76	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8s4__neon)
				77	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8s4__neon)
				78	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8s4__neon)
				79	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_8x8s4__neon)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	80
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	81	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8s4__neonfma)
				82	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8s4__neonfma)
				83	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8s4__neonfma)
				84	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_8x8s4__neonfma)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	85
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	86	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_ld64)
				87	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53)
				88	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55)
				89	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a75)
				90	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_pld_cortex_a75)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	91
Frank Barchard	3cb54f9	2020-04-10 10:46:08 -0700	[diff] [blame]	92	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_ld64)
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	93	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x12__aarch64_neonfma_cortex_a53)
				94	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53)
				95	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a57)
				96	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75)
				97	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x12__aarch64_neonfma_cortex_a53)
				98	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a53)
				99	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a55)
				100	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a57)
				101	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a75)
				102	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_ld128)
				103	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_ld64)
				104	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x8__aarch64_neonfma_cortex_a57)
				105	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x8__aarch64_neonfma_cortex_a75)
				106	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53)
				107	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55)
				108	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a73)
				109	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a57)
				110	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a75)
				111	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_ios)
				112	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_ld128)
				113	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_ld64)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	114
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	115	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__sse_load1)
				116	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__sse_load1)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	117
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	118	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__sse_dup)
				119	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__sse_dup)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	120
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	121	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8s4__sse)
				122	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8s4__sse)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	123
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	124	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x2c4__sse)
Marat Dukhan	b00004d	2020-02-13 09:28:35 -0800	[diff] [blame]	125
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	126	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__avx_broadcast)
				127	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__avx_broadcast)
				128	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x8__avx_broadcast)
				129	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__avx_broadcast)
				130	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_7x8__avx_broadcast)
				131	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x16__avx_broadcast)
				132	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_3x16__avx_broadcast)
				133	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x16__avx_broadcast)
				134	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x16__avx_broadcast)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	135
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	136	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__fma3_broadcast)
				137	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__fma3_broadcast)
				138	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x8__fma3_broadcast)
				139	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__fma3_broadcast)
				140	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_7x8__fma3_broadcast)
				141	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_8x8__fma3_broadcast)
				142	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x16__fma3_broadcast)
				143	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_3x16__fma3_broadcast)
				144	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x16__fma3_broadcast)
				145	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x16__fma3_broadcast)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	146
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	147	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x16s4__fma3_broadcast)
				148	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_3x16s4__fma3_broadcast)
				149	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x16s4__fma3_broadcast)
				150	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x16s4__fma3_broadcast)
Marat Dukhan	2712132	2019-12-09 14:57:40 -0800	[diff] [blame]	151
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	152	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x16__avx512f_broadcast)
				153	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x16__avx512f_broadcast)
				154	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_5x16__avx512f_broadcast)
				155	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x16__avx512f_broadcast)
				156	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_7x16__avx512f_broadcast)
				157	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_8x16__avx512f_broadcast)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	158
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	159	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__psimd_loadsplat)
				160	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__psimd_loadsplat)
				161	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__psimd_loadsplat)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	162
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	163	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8__psimd_splat)
				164	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8__psimd_splat)
				165	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8__psimd_splat)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	166
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	167	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x8s4__psimd)
				168	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x8s4__psimd)
				169	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_6x8s4__psimd)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	170
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	171	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x2c4__psimd)
Marat Dukhan	b00004d	2020-02-13 09:28:35 -0800	[diff] [blame]	172
Marat Dukhan	163a7e6	2020-04-09 04:19:26 -0700	[diff] [blame]	173	DECLARE_F32_GEMM_UKERNEL_FUNCTION(xnn_f32_gemm_ukernel_4x2__wasm)
				174	DECLARE_F32_GEMM_UKERNEL_FUNCTION(xnn_f32_gemm_ukernel_1x4__wasm)
				175	DECLARE_F32_GEMM_UKERNEL_FUNCTION(xnn_f32_gemm_ukernel_2x4__wasm)
				176	DECLARE_F32_GEMM_UKERNEL_FUNCTION(xnn_f32_gemm_ukernel_4x4__wasm)
				177
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	178	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x2__wasm)
				179	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x4__wasm)
				180	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_2x4__wasm)
				181	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x4__wasm)
Marat Dukhan	436ebe6	2019-12-04 15:10:12 -0800	[diff] [blame]	182
Marat Dukhan	163a7e6	2020-04-09 04:19:26 -0700	[diff] [blame]	183	DECLARE_F32_GEMM_UKERNEL_FUNCTION(xnn_f32_gemm_ukernel_4x2__scalar)
				184	DECLARE_F32_GEMM_UKERNEL_FUNCTION(xnn_f32_gemm_ukernel_1x4__scalar)
				185	DECLARE_F32_GEMM_UKERNEL_FUNCTION(xnn_f32_gemm_ukernel_2x4__scalar)
				186	DECLARE_F32_GEMM_UKERNEL_FUNCTION(xnn_f32_gemm_ukernel_4x4__scalar)
				187
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	188	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x2__scalar)
				189	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_1x4__scalar)
				190	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_2x4__scalar)
				191	DECLARE_F32_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemm_minmax_ukernel_4x4__scalar)
Marat Dukhan	436ebe6	2019-12-04 15:10:12 -0800	[diff] [blame]	192
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	193	#define DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(fn_name) \
				194	XNN_INTERNAL void fn_name( \
				195	size_t mr, \
				196	size_t nr, \
				197	size_t k, \
				198	const float* a, \
				199	size_t a_stride, \
				200	const float* w, \
				201	float* c, \
				202	size_t cm_stride, \
				203	size_t cn_stride, \
				204	const float* acc, \
Marat Dukhan	eb09a6b	2020-04-08 17:34:32 -0700	[diff] [blame]	205	const union xnn_f32_minmax_params* params);
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	206
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	207	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__neon_lane_ld64)
				208	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__neon_lane_ld64)
				209	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__neon_lane_ld128)
				210	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x8__neon_lane_ld64)
				211	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__neon_lane_ld64)
				212	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__neon_lane_ld128)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	213
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	214	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__neonfma_lane_ld64)
				215	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__neonfma_lane_ld64)
				216	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__neonfma_lane_ld128)
				217	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x8__neonfma_lane_ld64)
				218	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__neonfma_lane_ld64)
				219	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__neonfma_lane_ld128)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	220
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	221	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__neon_dup_ld64)
				222	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__neon_dup_ld64)
				223	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__neon_dup_ld128)
				224	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__neon_dup_ld64)
				225	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__neon_dup_ld128)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	226
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	227	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__neonfma_dup_ld64)
				228	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__neonfma_dup_ld64)
				229	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__neonfma_dup_ld128)
				230	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__neonfma_dup_ld64)
				231	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__neonfma_dup_ld128)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	232
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	233	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8s4__neon)
				234	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8s4__neon)
				235	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8s4__neon)
				236	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_8x8s4__neon)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	237
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	238	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8s4__neonfma)
				239	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8s4__neonfma)
				240	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8s4__neonfma)
				241	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_8x8s4__neonfma)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	242
Frank Barchard	3cb54f9	2020-04-10 10:46:08 -0700	[diff] [blame]	243	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__aarch64_neonfma_ld64)
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	244	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x12__aarch64_neonfma_cortex_a53)
				245	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53)
				246	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__aarch64_neonfma_cortex_a57)
				247	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75)
				248	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x12__aarch64_neonfma_cortex_a53)
				249	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__aarch64_neonfma_cortex_a53)
				250	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__aarch64_neonfma_cortex_a55)
				251	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__aarch64_neonfma_cortex_a57)
				252	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__aarch64_neonfma_cortex_a75)
				253	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__aarch64_neonfma_ld128)
				254	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__aarch64_neonfma_ld64)
				255	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x8__aarch64_neonfma_cortex_a57)
				256	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x8__aarch64_neonfma_cortex_a75)
				257	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53)
				258	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55)
				259	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__aarch64_neonfma_cortex_a73)
				260	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__aarch64_neonfma_cortex_a57)
				261	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__aarch64_neonfma_cortex_a75)
				262	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__aarch64_neonfma_ios)
				263	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__aarch64_neonfma_ld128)
				264	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__aarch64_neonfma_ld64)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	265
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	266	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__sse_load1)
				267	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__sse_load1)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	268
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	269	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__sse_dup)
				270	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__sse_dup)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	271
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	272	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8s4__sse)
				273	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8s4__sse)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	274
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	275	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__avx_broadcast)
				276	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__avx_broadcast)
				277	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x8__avx_broadcast)
				278	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__avx_broadcast)
				279	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_7x8__avx_broadcast)
				280	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x16__avx_broadcast)
				281	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_3x16__avx_broadcast)
				282	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x16__avx_broadcast)
				283	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x16__avx_broadcast)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	284
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	285	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__fma3_broadcast)
				286	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__fma3_broadcast)
				287	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x8__fma3_broadcast)
				288	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__fma3_broadcast)
				289	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_7x8__fma3_broadcast)
				290	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_8x8__fma3_broadcast)
				291	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x16__fma3_broadcast)
				292	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_3x16__fma3_broadcast)
				293	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x16__fma3_broadcast)
				294	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x16__fma3_broadcast)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	295
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	296	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x16s4__fma3_broadcast)
				297	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_3x16s4__fma3_broadcast)
				298	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x16s4__fma3_broadcast)
				299	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x16s4__fma3_broadcast)
Marat Dukhan	2712132	2019-12-09 14:57:40 -0800	[diff] [blame]	300
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	301	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x16__avx512f_broadcast)
				302	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x16__avx512f_broadcast)
				303	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_5x16__avx512f_broadcast)
				304	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x16__avx512f_broadcast)
				305	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_7x16__avx512f_broadcast)
				306	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_8x16__avx512f_broadcast)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	307
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	308	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__psimd_loadsplat)
				309	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__psimd_loadsplat)
				310	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__psimd_loadsplat)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	311
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	312	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8__psimd_splat)
				313	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8__psimd_splat)
				314	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8__psimd_splat)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	315
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	316	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x8s4__psimd)
				317	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x8s4__psimd)
				318	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_6x8s4__psimd)
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	319
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	320	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x4__wasm)
				321	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_2x4__wasm)
				322	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x2__wasm)
				323	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x4__wasm)
Marat Dukhan	436ebe6	2019-12-04 15:10:12 -0800	[diff] [blame]	324
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	325	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_1x4__scalar)
				326	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_2x4__scalar)
				327	DECLARE_F32_GEMMINC_MINMAX_UKERNEL_FUNCTION(xnn_f32_gemminc_minmax_ukernel_4x4__scalar)
Marat Dukhan	436ebe6	2019-12-04 15:10:12 -0800	[diff] [blame]	328
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	329	#define DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(fn_name) \
				330	void fn_name( \
				331	size_t mr, \
				332	size_t nr, \
				333	size_t k, \
				334	const void* a, \
				335	size_t a_stride, \
				336	const void* w, \
				337	void* c, \
				338	size_t cm_stride, \
				339	size_t cn_stride, \
Frank Barchard	875be77	2020-04-16 09:50:09 -0700	[diff] [blame]	340	const struct xnn_f16_scaleminmax_params* params);
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	341
Frank Barchard	36b76b6	2020-04-10 12:39:17 -0700	[diff] [blame]	342	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_1x16__aarch64_neonfp16arith_ld32)
				343	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_4x16__aarch64_neonfp16arith_ld32)
				344	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_6x16__aarch64_neonfp16arith_ld32)
Frank Barchard	bddfbcd	2020-04-15 12:32:41 -0700	[diff] [blame]	345	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_1x8__aarch64_neonfp16arith_ld64)
				346	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_4x8__aarch64_neonfp16arith_ld64)
				347	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_6x8__aarch64_neonfp16arith_ld64)
Frank Barchard	3b8e566	2020-04-20 12:12:53 -0700	[diff] [blame]	348	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_8x8__aarch64_neonfp16arith_ld64)
Frank Barchard	1f4e461	2020-04-13 18:24:54 -0700	[diff] [blame]	349	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_1x8__neonfp16arith_ld64)
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	350	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_4x8__neonfp16arith_ld64)
				351	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_6x8__neonfp16arith_ld64)
				352	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_8x8__neonfp16arith_ld64)
Frank Barchard	3f9f99f	2020-05-06 01:12:04 -0700	[diff] [blame^]	353	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_1x16__neonfp16arith_ld64)
				354	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_4x16__neonfp16arith_ld64)
				355	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_6x16__neonfp16arith_ld64)
				356	DECLARE_F16_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_f16_gemm_minmax_ukernel_8x16__neonfp16arith_ld64)
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	357
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	358	#define DECLARE_Q8_GEMM_MINMAX_UKERNEL_FUNCTION(fn_name) \
				359	XNN_INTERNAL void fn_name( \
				360	size_t mr, \
				361	size_t nr, \
				362	size_t k, \
				363	const uint8_t* a, \
				364	size_t a_stride, \
				365	const void* w, \
				366	uint8_t* c, \
				367	size_t cm_stride, \
				368	size_t cn_stride, \
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	369	const union xnn_q8_gemm_params* params);
				370
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	371
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	372	DECLARE_Q8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_q8_gemm_minmax_ukernel_4x8__neon)
				373	DECLARE_Q8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_q8_gemm_minmax_ukernel_8x8__neon)
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	374
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	375	DECLARE_Q8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_q8_gemm_minmax_ukernel_4x8__aarch32_neon)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	376
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	377	DECLARE_Q8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_q8_gemm_minmax_ukernel_8x8__aarch64_neon)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	378
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	379	DECLARE_Q8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_q8_gemm_minmax_ukernel_2x4c8__sse2)
				380	DECLARE_Q8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_q8_gemm_minmax_ukernel_4x4c2__sse2)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	381
Marat Dukhan	de06f49	2020-04-09 00:19:31 -0700	[diff] [blame]	382	DECLARE_Q8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_q8_gemm_minmax_ukernel_2x2__scalar)
Marat Dukhan	eccfd71	2019-12-08 16:49:27 -0800	[diff] [blame]	383
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	384	#ifdef __cplusplus
Marat Dukhan	80fc932	2019-09-29 21:06:36 -0700	[diff] [blame]	385	} // extern "C"
XNNPACK Team	b455b12	2019-09-27 18:10:33 -0700	[diff] [blame]	386	#endif