Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / XNNPACK / 60a3b19e9eb00c2cd0b4d5da83c820d6458478e7 / . / src / init.c
blob: 184741eb503ec9b3b47f74d162acdaf040649aff [file] [log] [blame]
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1// Copyright (c) Facebook, Inc. and its affiliates.
2// All rights reserved.
3//
4// Copyright 2019 Google LLC
5//
6// This source code is licensed under the BSD-style license found in the
7// LICENSE file in the root directory of this source tree.
8
Marat Dukhan01849012020-04-27 19:28:32 -0700[diff] [blame]9#include <math.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]10#include <stdbool.h>
11#include <stddef.h>
12#include <stdint.h>
Marat Dukhan04f03be2019-11-19 12:36:47 -0800[diff] [blame]13#include <string.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]14
Marat Dukhan57133c02020-04-13 00:54:59 -0700[diff] [blame]15#ifdef _WIN32
16 #include <windows.h>
17#else
18 #include <pthread.h>
19#endif
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]20
Marat Dukhand343c222019-10-07 09:22:14 -0700[diff] [blame]21#ifndef __EMSCRIPTEN__
22 #include <cpuinfo.h>
23#endif
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]24
25#include <xnnpack.h>
26#include <xnnpack/argmaxpool.h>
27#include <xnnpack/avgpool.h>
28#include <xnnpack/clamp.h>
Marat Dukhan1dadbf72019-10-01 10:46:20 -0700[diff] [blame]29#include <xnnpack/common.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]30#include <xnnpack/conv.h>
31#include <xnnpack/dwconv.h>
32#include <xnnpack/gavgpool.h>
33#include <xnnpack/gemm.h>
Marat Dukhan4662b192020-05-21 15:52:03 -0700[diff] [blame]34#include <xnnpack/fill.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]35#include <xnnpack/hswish.h>
Marat Dukhan660fd192020-03-10 04:55:30 -0700[diff] [blame]36#include <xnnpack/ibilinear.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]37#include <xnnpack/igemm.h>
38#include <xnnpack/log.h>
39#include <xnnpack/lut.h>
40#include <xnnpack/maxpool.h>
Marat Dukhan04f03be2019-11-19 12:36:47 -0800[diff] [blame]41#include <xnnpack/memory.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]42#include <xnnpack/pad.h>
43#include <xnnpack/params.h>
44#include <xnnpack/pavgpool.h>
45#include <xnnpack/prelu.h>
Marat Dukhan1edc4542020-01-27 12:40:13 -0800[diff] [blame]46#include <xnnpack/raddstoreexpminusmax.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]47#include <xnnpack/rmax.h>
48#include <xnnpack/spmm.h>
49#include <xnnpack/unpool.h>
50#include <xnnpack/vadd.h>
Marat Dukhan1e782c42019-11-21 17:02:40 -0800[diff] [blame]51#include <xnnpack/vbinary.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]52#include <xnnpack/vmulcaddc.h>
Marat Dukhan1e782c42019-11-21 17:02:40 -0800[diff] [blame]53#include <xnnpack/vunary.h>
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]54#include <xnnpack/zip.h>
55
56#ifndef XNN_ENABLE_ASSEMBLY
57 #define XNN_ENABLE_ASSEMBLY 1
58#endif
59
Marat Dukhan57133c02020-04-13 00:54:59 -0700[diff] [blame]60#ifdef _WIN32
61 static INIT_ONCE init_guard = INIT_ONCE_STATIC_INIT;
62#else
63 static pthread_once_t init_guard = PTHREAD_ONCE_INIT;
64#endif
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]65
66struct xnn_parameters xnn_params = {
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]67 .init_flags = 0
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]68};
69
Marat Dukhan01849012020-04-27 19:28:32 -0700[diff] [blame]70static void init(void) {
Marat Dukhana199d492020-07-24 15:01:25 -0700[diff] [blame]71#if XNN_ARCH_WASM || XNN_ARCH_WASMSIMD
Marat Dukhan01849012020-04-27 19:28:32 -0700[diff] [blame]72 // Unlike most other architectures, on x86/x86-64 when floating-point instructions
73 // have no NaN arguments, but produce NaN output, the output NaN has sign bit set.
74 // We use it to distinguish x86/x86-64 from other architectures, by doing subtraction
75 // of two infinities (must produce NaN per IEEE 754 standard).
76 static const volatile float inf = INFINITY;
77 const bool is_wasm_x86 = signbit(inf - inf);
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]78#endif
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]79 uint32_t init_flags = XNN_INIT_FLAG_XNNPACK;
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]80
Marat Dukhan1dadbf72019-10-01 10:46:20 -0700[diff] [blame]81#if XNN_ARCH_ARM
Frank Barchardbcdb1c12020-05-11 14:13:20 -0700[diff] [blame]82 #if XNN_PLATFORM_MOBILE
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]83 if (!cpuinfo_has_arm_neon()) {
84 xnn_log_error("XNNPACK initialization failed: NEON is not supported");
85 return;
86 }
87 #else
88 if (!cpuinfo_has_arm_vfpv2() && !cpuinfo_has_arm_vfpv3()) {
89 xnn_log_error("XNNPACK initialization failed: VFP is not supported");
90 return;
91 }
92 #endif
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]93
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]94 if (cpuinfo_has_arm_neon()) {
Marat Dukhanf28cddf2020-08-10 14:05:02 -0700[diff] [blame]95 /**************************** QS8 micro-kernels ****************************/
96 #ifndef XNN_NO_QS8_OPERATORS
97 init_flags |= XNN_INIT_FLAG_QS8;
98
Frank Barchard146e9992020-10-13 13:38:54 -0700[diff] [blame]99 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_2x8__neon_mlal_lane);
100 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_2x8__neon_mlal_lane);
Marat Dukhanf28cddf2020-08-10 14:05:02 -0700[diff] [blame]101 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x8__neon_mlal_lane);
102 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x8__neon_mlal_lane);
Frank Barchard146e9992020-10-13 13:38:54 -0700[diff] [blame]103 xnn_params.qs8.gemm.mr = 2;
Marat Dukhanf28cddf2020-08-10 14:05:02 -0700[diff] [blame]104 xnn_params.qs8.gemm.nr = 8;
105
106 xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x9__neon_mul16;
107 xnn_params.qs8.dwconv[0].channel_tile = 8;
108 xnn_params.qs8.dwconv[0].primary_tile = 9;
109
110 xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
111 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__neon_c8_acc2,
112 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__neon_c8_acc2,
113 .mr = 7,
114 };
Marat Dukhanff209482020-09-03 14:26:53 -0700[diff] [blame]115
116 xnn_params.qs8.vadd = (struct vbinary_parameters) {
117 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__neon_ld64_x8,
118 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x8,
119 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x8,
120 .element_tile = 8,
121 };
Marat Dukhanf28cddf2020-08-10 14:05:02 -0700[diff] [blame]122 #endif // XNN_NO_QS8_OPERATORS
123
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]124 /*************************** QU8 micro-kernels ***************************/
125 #ifndef XNN_NO_QU8_OPERATORS
126 init_flags |= XNN_INIT_FLAG_QU8;
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]127
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]128 xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_ukernel_4x8__neon);
129 xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_ukernel_4x8__neon);
130 xnn_params.qu8.gemm.mr = 4;
131 xnn_params.qu8.gemm.nr = 8;
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]132
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]133 xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_ukernel_up8x9__neon;
134 xnn_params.qu8.dwconv[0].channel_tile = 8;
135 xnn_params.qu8.dwconv[0].primary_tile = 9;
136 xnn_params.qu8.avgpool = (struct avgpool_parameters) {
137 .up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__neon_c8,
138 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__neon_c8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]139 .mr = 9,
140 .qr = 8,
141 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]142 xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
143 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__neon_c8,
144 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__neon_c8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]145 .mr = 7,
146 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]147 xnn_params.qu8.vadd = (xnn_vadd_ukernel_function) xnn_qu8_vadd_minmax_ukernel__neon;
148 #endif // XNN_NO_QU8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]149
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]150 /**************************** U8 micro-kernels ****************************/
151 #ifndef XNN_NO_U8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]152 init_flags |= XNN_INIT_FLAG_U8;
153
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]154 xnn_params.u8.maxpool = (struct maxpool_parameters) {
155 .ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__neon_c16,
156 .mr = 9,
157 .qr = 8,
158 };
159 xnn_params.u8.clamp = (xnn_univector_ukernel_function) xnn_u8_clamp_ukernel__neon_x64;
160 xnn_params.u8.rmax = xnn_u8_rmax_ukernel__neon;
161 xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
162 #endif // XNN_NO_U8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]163
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]164 /**************************** X8 micro-kernels ****************************/
165 #ifndef XNN_NO_X8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]166 init_flags |= XNN_INIT_FLAG_X8;
167
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]168 xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar;
169 xnn_params.x8.zip = (struct zip_parameters) {
170 .x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__neon,
171 .x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__neon,
172 .x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__neon,
173 .xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__neon,
174 };
175 #endif // XNN_NO_X8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]176
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]177 /**************************** F32 micro-kernels ****************************/
178 #ifndef XNN_NO_F32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]179 init_flags |= XNN_INIT_FLAG_F32;
180
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]181 #if XNN_ENABLE_ASSEMBLY
182 switch (cpuinfo_get_uarch(0)->uarch) {
183 case cpuinfo_uarch_cortex_a5:
184 case cpuinfo_uarch_cortex_a7:
Frank Barchard490febe2020-07-16 18:42:17 -0700[diff] [blame]185 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a7);
186 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a7);
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]187 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
188 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
189 xnn_params.f32.gemm.mr = 4;
190 xnn_params.f32.gemm.nr = 8;
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]191 break;
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]192
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]193 case cpuinfo_uarch_cortex_a53:
194 case cpuinfo_uarch_cortex_a55r0:
195 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53);
196 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53);
197 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
198 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
199 xnn_params.f32.gemm.mr = 4;
200 xnn_params.f32.gemm.nr = 8;
201 break;
202
203 case cpuinfo_uarch_cortex_a55:
204 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55);
205 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55);
206 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
207 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
208 xnn_params.f32.gemm.mr = 4;
209 xnn_params.f32.gemm.nr = 8;
210 break;
211
212 case cpuinfo_uarch_cortex_a57:
213 case cpuinfo_uarch_cortex_a72:
214 case cpuinfo_uarch_cortex_a73:
215 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_pld_cortex_a75);
216 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_pld_cortex_a75);
217 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
218 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
219 xnn_params.f32.gemm.mr = 4;
220 xnn_params.f32.gemm.nr = 8;
221 break;
222
223 case cpuinfo_uarch_krait:
224 default:
225 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a75);
226 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a75);
227 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
228 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
229 xnn_params.f32.gemm.mr = 4;
230 xnn_params.f32.gemm.nr = 8;
231 break;
232 }
233 #if XNN_MAX_UARCH_TYPES > 1
234 {
235 /* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
236 const uint32_t mr = xnn_params.f32.gemm.mr;
237 const uint32_t nr = xnn_params.f32.gemm.nr;
238 const uint32_t log2_sr = xnn_params.f32.gemm.log2_sr;
239 for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
240 const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
241 if (uarch_info == NULL) {
242 /* No more microarchitectures in the system */
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]243 break;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]244 }
245
246 switch (uarch_info->uarch) {
247 case cpuinfo_uarch_cortex_a53:
248 case cpuinfo_uarch_cortex_a55r0:
249 if (mr == 4 && nr == 8 && log2_sr == 0) {
250 xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53;
251 xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a53;
252 xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64;
253 xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64;
254 }
255 break;
256 case cpuinfo_uarch_cortex_a55:
257 if (mr == 4 && nr == 8 && log2_sr == 0) {
258 xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55;
259 xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch32_neon_cortex_a55;
260 xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64;
261 xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64;
262 }
263 break;
264 default:
265 break;
266 }
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]267 }
268 }
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]269 #endif // XNN_MAX_UARCH_TYPES > 1
270 #else // XNN_ENABLE_ASSEMBLY
271 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__neon_lane_ld128);
272 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__neon_lane_ld128);
273 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);
274 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neon_lane_ld64);
275 xnn_params.f32.gemm.mr = 4;
276 xnn_params.f32.gemm.nr = 8;
277 #endif // XNN_ENABLE_ASSEMBLY
278 xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__neon_lane_ld64);
279 xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__neon_lane_ld64);
280 xnn_params.f32.gemm2.mr = 4;
281 xnn_params.f32.gemm2.nr = 2;
282
283 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x4__neon;
284 xnn_params.f32.dwconv[0].channel_tile = 4,
285 xnn_params.f32.dwconv[0].primary_tile = 4,
286
287 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x9__neon;
288 xnn_params.f32.dwconv[1].channel_tile = 4;
289 xnn_params.f32.dwconv[1].primary_tile = 9;
290
291 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x25__neon_acc2;
292 xnn_params.f32.dwconv[2].channel_tile = 4;
293 xnn_params.f32.dwconv[2].primary_tile = 25;
294
295 xnn_params.f32.avgpool = (struct avgpool_parameters) {
296 .up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__neon_c4,
297 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__neon_c4,
298 .mr = 9,
299 .qr = 8,
300 };
301 xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
302 .up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__neon_c4,
303 .mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__neon_c4,
304 .mr = 9,
305 .qr = 8,
306 };
307 xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
308 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__neon_c4,
309 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__neon_c4,
310 .mr = 7,
311 };
312 xnn_params.f32.maxpool = (struct maxpool_parameters) {
313 .ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__neon_c4,
314 .mr = 9,
315 .qr = 8,
316 };
317 xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhanef25c6d2020-07-24 00:59:40 -0700[diff] [blame]318 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__neon_c4,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]319 .mr = 4,
320 };
321 xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhanef25c6d2020-07-24 00:59:40 -0700[diff] [blame]322 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__neon_c4,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]323 .mr = 9,
324 };
325 xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhanef25c6d2020-07-24 00:59:40 -0700[diff] [blame]326 .mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__neon_c4,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]327 .mr = 9,
328 .qr = 8,
329 };
330 xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
331 .ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__neon_c8,
332 .pixel_tile = 1,
333 .channel_tile = 8,
334 };
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]335 xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__neon_x8;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]336 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__neon_x8;
Marat Dukhan55dde5b2020-07-10 22:48:54 -0700[diff] [blame]337 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__neon_x16;
Marat Dukhan28813332020-06-10 18:05:38 -0700[diff] [blame]338 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__neon_x8;
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]339 xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__neon_x8;
Marat Dukhan64e52512020-06-09 13:41:16 -0700[diff] [blame]340 if (cpuinfo_has_arm_neon_v8()) {
341 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__neonv8_x8;
342 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__neonv8_x8;
343 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__neonv8_x8;
344 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__neonv8_x8;
345 } else {
346 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__neon_x8;
347 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__neon_x8;
348 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__neon_x8;
349 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__neon_x8;
350 }
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]351 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__neon_rr2_lut64_p2_nr2recps_x8;
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]352 xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__neon_x8;
Marat Dukhan6804bbd2020-06-30 19:26:11 -0700[diff] [blame]353 xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__scalar_sqrt_x1;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]354 xnn_params.f32.prelu = (struct prelu_parameters) {
355 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__neon_2x8,
356 .row_tile = 2,
357 .channel_tile = 8,
358 };
359 xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__neon_lut64_p2_x8;
360 xnn_params.f32.rmax = xnn_f32_rmax_ukernel__neon;
361 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]362 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__neon_x8,
363 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__neon_x8,
364 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__neon_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]365 .element_tile = 8,
366 };
367 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]368 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__scalar_x2,
369 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__scalar_x2,
370 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__scalar_x2,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]371 .element_tile = 2,
372 };
373 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]374 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__neon_x8,
375 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__neon_x8,
376 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__neon_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]377 .element_tile = 8,
378 };
379 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]380 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__neon_x8,
381 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__neon_x8,
382 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__neon_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]383 .element_tile = 8,
384 };
385 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]386 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__neon_x8,
387 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__neon_x8,
388 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__neon_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]389 .element_tile = 8,
390 };
391 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]392 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__neon_x8,
393 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__neon_x8,
394 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__neon_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]395 .element_tile = 8,
396 };
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]397 xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]398 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__neon_x8,
399 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__neon_x8,
400 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__neon_x8,
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]401 .element_tile = 8,
402 };
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]403 xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan9531e9f2020-07-24 15:25:02 -0700[diff] [blame]404 .ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__neon_2x,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]405 .channel_tile = 4,
406 .row_tile = 2,
407 };
408 #endif // XNN_NO_F32_OPERATORS
409
410 /**************************** X32 micro-kernels ****************************/
411 #ifndef XNN_NO_X32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]412 init_flags |= XNN_INIT_FLAG_X32;
413
Marat Dukhan4662b192020-05-21 15:52:03 -0700[diff] [blame]414 xnn_params.x32.fill = (struct fill_parameters) {
415 .ukernel = (xnn_fill_ukernel_function) xnn_x32_fill_ukernel__neon,
416 .row_tile = 1,
417 };
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]418 xnn_params.x32.pad = (struct pad_parameters) {
Marat Dukhan63523d42020-05-22 17:07:33 -0700[diff] [blame]419 .ukernel = (xnn_pad_ukernel_function) xnn_x32_pad_ukernel__neon,
420 .row_tile = 1,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]421 };
422 xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__neon;
423 xnn_params.x32.zip = (struct zip_parameters) {
424 .x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__neon,
425 .x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__neon,
426 .x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__neon,
427 .xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__neon,
428 };
429 #endif // XNN_NO_X32_OPERATORS
430 } else if (!XNN_PLATFORM_MOBILE) {
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]431 /*************************** QU8 micro-kernels ***************************/
432 #ifndef XNN_NO_QU8_OPERATORS
433 init_flags |= XNN_INIT_FLAG_QU8;
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]434
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]435 xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_ukernel_2x2__scalar);
436 xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_ukernel_2x2__scalar);
437 xnn_params.qu8.gemm.mr = 2;
438 xnn_params.qu8.gemm.nr = 2;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]439
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]440 xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_ukernel_up1x9__scalar;
441 xnn_params.qu8.dwconv[0].channel_tile = 1;
442 xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]443
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]444 xnn_params.qu8.avgpool = (struct avgpool_parameters) {
445 .up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__scalar_c1,
446 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]447 .mr = 9,
448 .qr = 8,
449 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]450 xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
451 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__scalar_c1,
452 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]453 .mr = 7,
454 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]455 xnn_params.qu8.vadd = (xnn_vadd_ukernel_function) xnn_qu8_vadd_minmax_ukernel__scalar;
456 #endif // XNN_NO_QU8_OPERATORS
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]457
458 /**************************** U8 micro-kernels ****************************/
459 #ifndef XNN_NO_U8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]460 init_flags |= XNN_INIT_FLAG_U8;
461
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]462 xnn_params.u8.maxpool = (struct maxpool_parameters) {
463 .ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__scalar_c1,
464 .mr = 9,
465 .qr = 8,
466 };
467 xnn_params.u8.clamp = (xnn_univector_ukernel_function) xnn_u8_clamp_ukernel__scalar_x4;
468 xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
469 xnn_params.u8.rmax = xnn_u8_rmax_ukernel__scalar;
470 #endif // XNN_NO_U8_OPERATORS
471
472 /**************************** X8 micro-kernels ****************************/
473 #ifndef XNN_NO_X8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]474 init_flags |= XNN_INIT_FLAG_X8;
475
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]476 xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar;
477 xnn_params.x8.zip = (struct zip_parameters) {
478 .x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__scalar,
479 .x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__scalar,
480 .x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__scalar,
481 .xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__scalar,
482 };
483 #endif // XNN_NO_X8_OPERATORS
484
485 /**************************** F32 micro-kernels ****************************/
486 #ifndef XNN_NO_F32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]487 init_flags |= XNN_INIT_FLAG_F32;
488
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]489 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x4__scalar);
490 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x4__scalar);
491 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x4__scalar);
492 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x4__scalar);
Marat Dukhan467f6362020-05-22 23:21:55 -0700[diff] [blame]493 xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_4x4__scalar);
494 xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_4x4__scalar);
495 xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x4__scalar);
496 xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x4__scalar);
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]497 xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x4__scalar);
498 xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x4__scalar);
499 xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x4__scalar);
500 xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x4__scalar);
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]501 xnn_params.f32.gemm.mr = 4;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]502 xnn_params.f32.gemm.nr = 4;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]503
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]504 xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__scalar);
505 xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__scalar),
506 xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2__scalar);
507 xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2__scalar),
508 xnn_params.f32.gemm2.mr = 4;
509 xnn_params.f32.gemm2.nr = 2;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]510
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]511 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x4__scalar_acc2;
512 xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x4__scalar_acc2;
513 xnn_params.f32.dwconv[0].channel_tile = 1;
514 xnn_params.f32.dwconv[0].primary_tile = 4;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]515
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]516 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x9__scalar_acc2;
517 xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x9__scalar_acc2;
518 xnn_params.f32.dwconv[1].channel_tile = 1;
519 xnn_params.f32.dwconv[1].primary_tile = 9;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]520
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]521 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x25__scalar_acc2;
522 xnn_params.f32.dwconv[2].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x25__scalar_acc2;
523 xnn_params.f32.dwconv[2].channel_tile = 1;
524 xnn_params.f32.dwconv[2].primary_tile = 25;
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]525
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]526 xnn_params.f32.avgpool = (struct avgpool_parameters) {
527 .up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__scalar_c1,
528 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__scalar_c1,
529 .mr = 9,
530 .qr = 8,
531 };
532 xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
533 .up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__scalar_c1,
534 .mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__scalar_c1,
535 .mr = 9,
536 .qr = 8,
537 };
538 xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
539 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__scalar_c1,
540 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__scalar_c1,
541 .mr = 7,
542 };
543 xnn_params.f32.maxpool = (struct maxpool_parameters) {
544 .ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__scalar_c1,
545 .mr = 9,
546 .qr = 8,
547 };
548 xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
549 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__scalar_c1,
550 .mr = 4,
551 };
552 xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
553 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__scalar_c1,
554 .mr = 9,
555 };
556 xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
557 .mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__scalar_c1,
558 .mr = 9,
559 .qr = 8,
560 };
561 xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
562 .ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__scalar_c2,
563 .pixel_tile = 1,
564 .channel_tile = 2,
565 };
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]566 xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__scalar_x4;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]567 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__scalar_x4;
568 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__scalar_x4;
Marat Dukhan28813332020-06-10 18:05:38 -0700[diff] [blame]569 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__scalar_x4;
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]570 xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__scalar_x4;
Marat Dukhan64e52512020-06-09 13:41:16 -0700[diff] [blame]571 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__scalar_libm_x1;
572 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__scalar_libm_x1;
573 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__scalar_libm_x1;
574 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__scalar_libm_x1;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]575 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__scalar_lut64_p2_div_x2;
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]576 xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__scalar_x4;
Marat Dukhan6804bbd2020-06-30 19:26:11 -0700[diff] [blame]577 xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__scalar_sqrt_x1;
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]578 xnn_params.f32.prelu = (struct prelu_parameters) {
579 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__scalar_2x4,
580 .row_tile = 4,
581 .channel_tile = 4,
582 };
583 xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__scalar_p5_x4_acc2;
584 xnn_params.f32.rmax = xnn_f32_rmax_ukernel__scalar;
585 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]586 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__scalar_x8,
587 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__scalar_x8,
588 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__scalar_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]589 .element_tile = 8,
590 };
591 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]592 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__scalar_x2,
593 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__scalar_x2,
594 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__scalar_x2,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]595 .element_tile = 2,
596 };
597 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]598 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__scalar_x8,
599 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__scalar_x8,
600 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__scalar_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]601 .element_tile = 8,
602 };
603 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]604 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__scalar_x8,
605 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__scalar_x8,
606 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__scalar_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]607 .element_tile = 8,
608 };
609 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]610 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__scalar_x8,
611 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__scalar_x8,
612 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__scalar_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]613 .element_tile = 8,
614 };
615 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]616 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__scalar_x8,
617 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__scalar_x8,
618 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__scalar_x8,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]619 .element_tile = 8,
620 };
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]621 xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]622 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__scalar_x8,
623 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
624 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]625 .element_tile = 8,
626 };
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]627 xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan9531e9f2020-07-24 15:25:02 -0700[diff] [blame]628 .ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c1__scalar_2x,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]629 .channel_tile = 1,
630 .row_tile = 2,
631 };
632 #ifndef XNN_NO_NCHW_OPERATORS
633 xnn_params.f32.spmm = (struct spmm_parameters) {
634 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x1__scalar,
635 .mr = 8,
636 .nr = 1,
637 };
638 xnn_params.f32.spmm2 = (struct spmm_parameters) {
639 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x2__scalar,
640 .mr = 8,
641 .nr = 2,
642 };
643 xnn_params.f32.spmm4 = (struct spmm_parameters) {
644 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x4__scalar,
645 .mr = 8,
646 .nr = 4,
647 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]648 xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]649 .ukernel_with_symm_padding =
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]650 (xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__scalar_1x1,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]651 .output_channel_tile = 4,
652 .output_height_tile = 1,
653 .output_width_tile = 1,
654 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]655 xnn_params.f32.dwconv_chw_3x3 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]656 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3p1__scalar_1x1_acc3,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]657 .input_width_tile = 1,
658 .output_width_tile = 1,
659 .output_height_tile = 1,
660 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]661 xnn_params.f32.dwconv_chw_3x3s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]662 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3s2p1__scalar_1x1_acc3,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]663 .input_width_tile = 1,
664 .output_width_tile = 1,
665 .output_height_tile = 1,
666 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]667 xnn_params.f32.dwconv_chw_5x5 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]668 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_5x5p2__scalar_1x1_acc5,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]669 .input_width_tile = 1,
670 .output_width_tile = 1,
671 .output_height_tile = 1,
672 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]673 xnn_params.f32.dwconv_chw_5x5s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]674 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_5x5s2p2__scalar_1x1_acc5,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]675 .input_width_tile = 1,
676 .output_width_tile = 1,
677 .output_height_tile = 1,
678 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]679 xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
680 .ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__scalar_x1,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]681 .channel_tile = 1,
682 };
683 #endif // XNN_NO_NCHW_OPERATORS
684 #endif // XNN_NO_F32_OPERATORS
685
686 /**************************** X32 micro-kernels ****************************/
687 #ifndef XNN_NO_X32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]688 init_flags |= XNN_INIT_FLAG_X32;
689
Marat Dukhan4662b192020-05-21 15:52:03 -0700[diff] [blame]690 xnn_params.x32.fill = (struct fill_parameters) {
691 .ukernel = (xnn_fill_ukernel_function) xnn_x32_fill_ukernel__scalar_int,
692 .row_tile = 1,
693 };
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]694 xnn_params.x32.pad = (struct pad_parameters) {
Marat Dukhan63523d42020-05-22 17:07:33 -0700[diff] [blame]695 .ukernel = (xnn_pad_ukernel_function) xnn_x32_pad_ukernel__scalar_int,
696 .row_tile = 1,
Marat Dukhan3b745a42020-05-10 21:43:25 -0700[diff] [blame]697 };
698 xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__scalar;
699 xnn_params.x32.zip = (struct zip_parameters) {
700 .x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__scalar,
701 .x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__scalar,
702 .x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__scalar,
703 .xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__scalar,
704 };
705 #endif // XNN_NO_X32_OPERATORS
706 }
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]707
Marat Dukhan1dadbf72019-10-01 10:46:20 -0700[diff] [blame]708#elif XNN_ARCH_ARM64
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]709
Marat Dukhanf28cddf2020-08-10 14:05:02 -0700[diff] [blame]710 /**************************** QS8 micro-kernels ****************************/
711 #ifndef XNN_NO_QS8_OPERATORS
712 init_flags |= XNN_INIT_FLAG_QS8;
713
Marat Dukhan31677ad2020-10-13 23:59:31 -0700[diff] [blame]714 #if XNN_PLATFORM_IOS
Frank Barchardbc0c7292020-10-06 13:36:54 -0700[diff] [blame]715 #if XNN_ENABLE_ASSEMBLY
Marat Dukhan31677ad2020-10-13 23:59:31 -0700[diff] [blame]716 if (cpuinfo_has_arm_neon_dot()) {
717 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_4x16c4__aarch64_neondot_ld64);
718 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x16c4__aarch64_neondot_ld64);
719 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_4x16c4__neondot);
720 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x16c4__neondot);
721 xnn_params.qs8.gemm.mr = 4;
722 xnn_params.qs8.gemm.nr = 16;
723 xnn_params.qs8.gemm.log2_kr = 2;
724 } else {
725 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_2x8__neon_mlal_lane);
726 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_2x8__neon_mlal_lane);
727 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x8__neon_mlal_lane);
728 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x8__neon_mlal_lane);
729 xnn_params.qs8.gemm.mr = 2;
730 xnn_params.qs8.gemm.nr = 8;
Frank Barchard1e8590e2020-10-12 21:20:46 -0700[diff] [blame]731 }
Marat Dukhan31677ad2020-10-13 23:59:31 -0700[diff] [blame]732 #else // !XNN_ENABLE_ASSEMBLY
733 if (cpuinfo_has_arm_neon_dot()) {
734 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_4x16c4__neondot);
735 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x16c4__neondot);
736 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_4x16c4__neondot);
737 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x16c4__neondot);
738 xnn_params.qs8.gemm.mr = 4;
739 xnn_params.qs8.gemm.nr = 16;
740 xnn_params.qs8.gemm.log2_kr = 2;
741 } else {
742 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_2x8__neon_mlal_lane);
743 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_2x8__neon_mlal_lane);
744 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x8__neon_mlal_lane);
745 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x8__neon_mlal_lane);
746 xnn_params.qs8.gemm.mr = 2;
747 xnn_params.qs8.gemm.nr = 8;
748 }
749 #endif // XNN_ENABLE_ASSEMBLY
750 #else // !XNN_PLATFORM_IOS
751 #if XNN_ENABLE_ASSEMBLY
752 if (cpuinfo_has_arm_neon_dot()) {
753 switch (cpuinfo_get_core(0)->uarch) {
754 case cpuinfo_uarch_cortex_a55:
755 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_4x16c4__aarch64_neondot_cortex_a55);
756 break;
757 default:
758 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_4x16c4__aarch64_neondot_ld64);
759 break;
760 }
761 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_4x16c4__neondot);
762 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x16c4__aarch64_neondot_ld64);
763 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x16c4__neondot);
764 xnn_params.qs8.gemm.mr = 4;
765 xnn_params.qs8.gemm.nr = 16;
766 xnn_params.qs8.gemm.log2_kr = 2;
767 } else {
768 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_2x8__neon_mlal_lane);
769 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_2x8__neon_mlal_lane);
770 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x8__neon_mlal_lane);
771 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x8__neon_mlal_lane);
772 xnn_params.qs8.gemm.mr = 2;
773 xnn_params.qs8.gemm.nr = 8;
774 }
775 #if XNN_MAX_UARCH_TYPES > 1
776 {
777 /* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
778 const uint32_t mr = xnn_params.qs8.gemm.mr;
779 const uint32_t nr = xnn_params.qs8.gemm.nr;
780 const uint32_t log2_kr = xnn_params.qs8.gemm.log2_kr;
781 for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
782 const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
783 if (uarch_info == NULL) {
784 /* No more microarchitectures in the system */
785 break;
786 }
787
788 switch (uarch_info->uarch) {
789 case cpuinfo_uarch_cortex_a55:
790 if (mr == 4 && nr == 16 && log2_kr == 2) {
791 xnn_params.qs8.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_4x16c4__aarch64_neondot_cortex_a55;
792 xnn_params.qs8.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_4x16c4__neondot;
793 xnn_params.qs8.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x16c4__aarch64_neondot_ld64;
794 xnn_params.qs8.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x16c4__neondot;
795 }
796 break;
797 default:
798 break;
799 }
800 }
801 }
802 #endif // XNN_MAX_UARCH_TYPES > 1
803 #else // !XNN_ENABLE_ASSEMBLY
804 if (cpuinfo_has_arm_neon_dot()) {
805 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_4x16c4__neondot);
806 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x16c4__neondot);
807 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_4x16c4__neondot);
808 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x16c4__neondot);
809 xnn_params.qs8.gemm.mr = 4;
810 xnn_params.qs8.gemm.nr = 16;
811 xnn_params.qs8.gemm.log2_kr = 2;
812 } else {
813 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_2x8__neon_mlal_lane);
814 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_2x8__neon_mlal_lane);
815 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x8__neon_mlal_lane);
816 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x8__neon_mlal_lane);
817 xnn_params.qs8.gemm.mr = 2;
818 xnn_params.qs8.gemm.nr = 8;
819 }
820 #endif // XNN_ENABLE_ASSEMBLY
821 #endif // XNN_PLATFORM_IOS
Marat Dukhanf28cddf2020-08-10 14:05:02 -0700[diff] [blame]822
823 xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x9__neon_mul16;
824 xnn_params.qs8.dwconv[0].channel_tile = 8;
825 xnn_params.qs8.dwconv[0].primary_tile = 9;
826
827 xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
828 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__neon_c8_acc2,
829 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__neon_c8_acc2,
830 .mr = 7,
831 };
Marat Dukhanff209482020-09-03 14:26:53 -0700[diff] [blame]832
833 xnn_params.qs8.vadd = (struct vbinary_parameters) {
834 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__neon_ld64_x8,
835 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x8,
836 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__neon_ld64_x8,
837 .element_tile = 8,
838 };
Marat Dukhanf28cddf2020-08-10 14:05:02 -0700[diff] [blame]839 #endif // XNN_NO_QS8_OPERATORS
840
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]841 /**************************** QU8 micro-kernels ****************************/
842 #ifndef XNN_NO_QU8_OPERATORS
843 init_flags |= XNN_INIT_FLAG_QU8;
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]844
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]845 xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_ukernel_8x8__neon);
846 xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_ukernel_8x8__neon);
847 xnn_params.qu8.gemm.mr = 8;
848 xnn_params.qu8.gemm.nr = 8;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]849
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]850 xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_ukernel_up8x9__neon;
851 xnn_params.qu8.dwconv[0].channel_tile = 8;
852 xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]853
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]854 xnn_params.qu8.avgpool = (struct avgpool_parameters) {
855 .up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__neon_c8,
856 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__neon_c8,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]857 .mr = 9,
858 .qr = 8,
859 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]860 xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
861 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__neon_c8,
862 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__neon_c8,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]863 .mr = 7,
864 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]865 xnn_params.qu8.vadd = (xnn_vadd_ukernel_function) xnn_qu8_vadd_minmax_ukernel__neon;
866 #endif // XNN_NO_QU8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]867
868 /**************************** U8 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]869 #ifndef XNN_NO_U8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]870 init_flags |= XNN_INIT_FLAG_U8;
871
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]872 xnn_params.u8.maxpool = (struct maxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]873 .ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__neon_c16,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]874 .mr = 9,
875 .qr = 8,
876 };
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]877 xnn_params.u8.clamp = (xnn_univector_ukernel_function) xnn_u8_clamp_ukernel__neon_x64;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]878 xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
879 xnn_params.u8.rmax = xnn_u8_rmax_ukernel__neon;
880 #endif // XNN_NO_U8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]881
882 /**************************** X8 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]883 #ifndef XNN_NO_X8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]884 init_flags |= XNN_INIT_FLAG_X8;
885
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]886 xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar;
887 xnn_params.x8.zip = (struct zip_parameters) {
888 .x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__neon,
889 .x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__neon,
890 .x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__neon,
891 .xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__neon,
892 };
893 #endif // XNN_NO_X8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]894
Frank Barchard7e2cbb02020-06-12 01:22:13 -0700[diff] [blame]895 /**************************** F16 micro-kernels ****************************/
896 #ifndef XNN_NO_F16_OPERATORS
Marat Dukhan8d5d2592020-06-19 12:48:57 -0700[diff] [blame]897 if (cpuinfo_has_arm_neon_fp16_arith()) {
898 init_flags |= XNN_INIT_FLAG_F16;
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]899
Frank Barchard6b73c4f2020-06-26 18:40:40 -0700[diff] [blame]900 #if XNN_ENABLE_ASSEMBLY
901 xnn_params.f16.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__aarch64_neonfp16arith_ld32);
902 xnn_params.f16.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_1x16__aarch64_neonfp16arith_ld32);
903 #else
904 xnn_params.f16.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_6x16__neonfp16arith_ld64);
905 xnn_params.f16.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f16_gemm_minmax_ukernel_1x16__neonfp16arith_ld64);
906 #endif
Frank Barchard49b4dcc2020-06-26 14:07:19 -0700[diff] [blame]907 xnn_params.f16.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f16_igemm_minmax_ukernel_6x16__neonfp16arith_ld64);
Frank Barchard49b4dcc2020-06-26 14:07:19 -0700[diff] [blame]908 xnn_params.f16.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f16_igemm_minmax_ukernel_1x16__neonfp16arith_ld64);
909 xnn_params.f16.gemm.mr = 6;
910 xnn_params.f16.gemm.nr = 16;
911
912 xnn_params.f16.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f16_dwconv_minmax_ukernel_up16x4__neonfp16arith;
913 xnn_params.f16.dwconv[0].channel_tile = 16;
914 xnn_params.f16.dwconv[0].primary_tile = 4;
915
916 xnn_params.f16.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f16_dwconv_minmax_ukernel_up16x9__neonfp16arith;
917 xnn_params.f16.dwconv[1].channel_tile = 16;
918 xnn_params.f16.dwconv[1].primary_tile = 9;
919
920 xnn_params.f16.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f16_dwconv_minmax_ukernel_up8x25__neonfp16arith_acc2;
921 xnn_params.f16.dwconv[2].channel_tile = 8;
922 xnn_params.f16.dwconv[2].primary_tile = 25;
923
Marat Dukhan8d5d2592020-06-19 12:48:57 -0700[diff] [blame]924 xnn_params.f16.gavgpool = (struct gavgpool_parameters) {
925 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_f16_gavgpool_minmax_ukernel_7x__neonfp16arith_c8,
926 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f16_gavgpool_minmax_ukernel_7p7x__neonfp16arith_c8,
927 .mr = 7,
928 };
Frank Barchard01898c02020-06-23 21:49:50 -0700[diff] [blame]929 xnn_params.f16.vadd = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]930 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vadd_minmax_ukernel__neonfp16arith_x16,
931 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vaddc_minmax_ukernel__neonfp16arith_x16,
932 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vaddc_minmax_ukernel__neonfp16arith_x16,
Frank Barchard01898c02020-06-23 21:49:50 -0700[diff] [blame]933 .element_tile = 16,
934 };
Frank Barchard0ea6a772020-09-09 15:26:31 -0700[diff] [blame]935 xnn_params.f16.vmul = (struct vbinary_parameters) {
936 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vmul_minmax_ukernel__neonfp16arith_x16,
937 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vmulc_minmax_ukernel__neonfp16arith_x16,
938 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f16_vmulc_minmax_ukernel__neonfp16arith_x16,
939 .element_tile = 16,
940 };
Frank Barchard49b4dcc2020-06-26 14:07:19 -0700[diff] [blame]941 xnn_params.f16.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan9531e9f2020-07-24 15:25:02 -0700[diff] [blame]942 .ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f16_vmulcaddc_minmax_ukernel_c8__neonfp16arith_2x,
Frank Barchard49b4dcc2020-06-26 14:07:19 -0700[diff] [blame]943 .channel_tile = 8,
944 .row_tile = 2,
945 };
Frank Barcharda96948e2020-09-11 15:34:18 -0700[diff] [blame]946 xnn_params.f16.hswish = (xnn_univector_ukernel_function) xnn_f16_hswish_ukernel__neonfp16arith_x16;
Marat Dukhan8d5d2592020-06-19 12:48:57 -0700[diff] [blame]947 }
Frank Barchard7e2cbb02020-06-12 01:22:13 -0700[diff] [blame]948 #endif // XNN_NO_F16_OPERATORS
949
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]950 /**************************** F32 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]951 #ifndef XNN_NO_F32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]952 init_flags |= XNN_INIT_FLAG_F32;
953
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]954 #if XNN_PLATFORM_IOS
955 #if XNN_ENABLE_ASSEMBLY
Frank Barchard016e5862020-06-11 12:24:50 -0700[diff] [blame]956 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a75);
957 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a75);
958 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
959 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]960 xnn_params.f32.gemm.mr = 6;
961 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]962 #else // !XNN_ENABLE_ASSEMBLY
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]963 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__neonfma_lane_ld64);
964 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__neonfma_lane_ld64);
965 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neonfma_lane_ld64);
966 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neonfma_lane_ld64);
967 xnn_params.f32.gemm.mr = 6;
968 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]969 #endif // XNN_ENABLE_ASSEMBLY
970 #else // !XNN_PLATFORM_IOS
971 #if XNN_ENABLE_ASSEMBLY
972 switch (cpuinfo_get_core(0)->uarch) {
973 case cpuinfo_uarch_cortex_a57:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]974 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a57);
975 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a57);
976 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a57);
977 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a57);
978 xnn_params.f32.gemm.mr = 6;
979 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]980 break;
981 case cpuinfo_uarch_cortex_a72:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]982 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a75);
983 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a75);
984 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
985 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
986 xnn_params.f32.gemm.mr = 4;
987 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]988 break;
989 case cpuinfo_uarch_cortex_a75:
990 case cpuinfo_uarch_cortex_a76:
991 case cpuinfo_uarch_exynos_m3:
992 case cpuinfo_uarch_exynos_m4:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]993 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a75);
994 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a75);
995 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
996 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
997 xnn_params.f32.gemm.mr = 6;
998 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]999 break;
1000 case cpuinfo_uarch_exynos_m1:
1001 case cpuinfo_uarch_exynos_m2:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1002 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8s4__neonfma);
1003 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8s4__neonfma);
1004 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8s4__neonfma);
1005 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8s4__neonfma);
1006 xnn_params.f32.gemm.mr = 6;
1007 xnn_params.f32.gemm.nr = 8;
1008 xnn_params.f32.gemm.log2_sr = 2;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1009 break;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1010 case cpuinfo_uarch_cortex_a53:
1011 case cpuinfo_uarch_cortex_a55r0:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1012 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53);
1013 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53);
1014 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53);
1015 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53);
1016 xnn_params.f32.gemm.mr = 6;
1017 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1018 break;
1019 case cpuinfo_uarch_cortex_a55:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1020 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55);
1021 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55);
1022 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53);
1023 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53);
1024 xnn_params.f32.gemm.mr = 6;
1025 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1026 break;
1027 case cpuinfo_uarch_cortex_a73:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1028 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a73);
1029 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a73);
1030 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
1031 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75);
1032 xnn_params.f32.gemm.mr = 6;
1033 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1034 break;
1035 default:
1036 case cpuinfo_uarch_cortex_a77:
1037 case cpuinfo_uarch_exynos_m5:
1038 case cpuinfo_uarch_kryo:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1039 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a57);
1040 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a57);
1041 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a57);
1042 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a57);
1043 xnn_params.f32.gemm.mr = 4;
1044 xnn_params.f32.gemm.nr = 8;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1045 break;
1046 }
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]1047 #if XNN_MAX_UARCH_TYPES > 1
1048 {
1049 /* Choose micro-kernels for little cores according to micro-kernel specification for the big core */
1050 const uint32_t mr = xnn_params.f32.gemm.mr;
1051 const uint32_t nr = xnn_params.f32.gemm.nr;
1052 const uint32_t log2_sr = xnn_params.f32.gemm.log2_sr;
1053 for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {
1054 const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i);
1055 if (uarch_info == NULL) {
1056 /* No more microarchitectures in the system */
1057 break;
1058 }
1059
1060 switch (uarch_info->uarch) {
1061 case cpuinfo_uarch_cortex_a53:
1062 case cpuinfo_uarch_cortex_a55r0:
1063 if (mr == 6 && nr == 8 && log2_sr == 0) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1064 xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53;
1065 xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a53;
1066 xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
1067 xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]1068 } else if (mr == 4 && nr == 8 && log2_sr == 0) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1069 xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a53;
1070 xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a53;
1071 xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
1072 xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]1073 }
1074 break;
1075 case cpuinfo_uarch_cortex_a55:
1076 if (mr == 6 && nr == 8 && log2_sr == 0) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1077 xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55;
1078 xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__aarch64_neonfma_cortex_a55;
1079 xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
1080 xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]1081 } else if (mr == 4 && nr == 8 && log2_sr == 0) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1082 xnn_params.f32.gemm.minmax.gemm.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a55;
1083 xnn_params.f32.gemm.minmax.igemm.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a55;
1084 xnn_params.f32.gemm.minmax.gemm1.function[i] = (xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
1085 xnn_params.f32.gemm.minmax.igemm1.function[i] = (xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a53;
Marat Dukhan05702cf2020-03-26 15:41:33 -0700[diff] [blame]1086 }
1087 break;
1088 default:
1089 break;
1090 }
1091 }
1092 }
1093 #endif // XNN_MAX_UARCH_TYPES > 1
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1094 #else // !XNN_ENABLE_ASSEMBLY
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1095 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_6x8__neonfma_lane_ld64);
1096 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_6x8__neonfma_lane_ld64);
1097 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__neonfma_lane_ld64);
1098 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__neonfma_lane_ld64);
1099 xnn_params.f32.gemm.mr = 6;
1100 xnn_params.f32.gemm.nr = 8;
Marat Dukhan31677ad2020-10-13 23:59:31 -0700[diff] [blame]1101 #endif // XNN_ENABLE_ASSEMBLY
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1102 #endif // XNN_PLATFORM_IOS
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1103 xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__neonfma_lane_ld64);
1104 xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__neonfma_lane_ld64);
1105 xnn_params.f32.gemm2.mr = 4;
1106 xnn_params.f32.gemm2.nr = 2;
1107
Marat Dukhanf5425ea2020-04-24 01:46:00 -0700[diff] [blame]1108 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x4__neonfma;
1109 xnn_params.f32.dwconv[0].channel_tile = 8;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1110 xnn_params.f32.dwconv[0].primary_tile = 4;
1111
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1112 #if XNN_PLATFORM_IOS
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1113 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__neonfma;
1114 xnn_params.f32.dwconv[1].channel_tile = 8;
1115 xnn_params.f32.dwconv[1].primary_tile = 9;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1116 #else // !XNN_PLATFORM_IOS
1117 switch (cpuinfo_get_core(0)->uarch) {
1118 case cpuinfo_uarch_kryo:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1119 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x9__neonfma;
1120 xnn_params.f32.dwconv[1].channel_tile = 4;
1121 xnn_params.f32.dwconv[1].primary_tile = 9;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1122 break;
1123 #if XNN_ENABLE_ASSEMBLY
1124 case cpuinfo_uarch_cortex_a53:
1125 case cpuinfo_uarch_cortex_a55r0:
1126 case cpuinfo_uarch_cortex_a55:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1127 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x9__aarch64_neonfma_cortex_a55;
1128 xnn_params.f32.dwconv[1].channel_tile = 4;
1129 xnn_params.f32.dwconv[1].primary_tile = 9;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1130 break;
1131 #endif // XNN_ENABLE_ASSEMBLY
1132 default:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1133 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__neonfma;
1134 xnn_params.f32.dwconv[1].channel_tile = 8;
1135 xnn_params.f32.dwconv[1].primary_tile = 9;
Frank Barchard0d1052c2020-03-23 17:28:13 -0700[diff] [blame]1136 break;
1137 }
1138 #endif // XNN_PLATFORM_IOS
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1139
Marat Dukhanf5425ea2020-04-24 01:46:00 -0700[diff] [blame]1140 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x25__neonfma_acc2;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1141 xnn_params.f32.dwconv[2].channel_tile = 4;
1142 xnn_params.f32.dwconv[2].primary_tile = 25;
1143
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1144 xnn_params.f32.avgpool = (struct avgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1145 .up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__neon_c4,
1146 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__neon_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1147 .mr = 9,
1148 .qr = 8,
1149 };
1150 xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1151 .up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__neon_c4,
1152 .mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__neon_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1153 .mr = 9,
1154 .qr = 8,
1155 };
1156 xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1157 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__neon_c4,
1158 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__neon_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1159 .mr = 7,
1160 };
1161 xnn_params.f32.maxpool = (struct maxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1162 .ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__neon_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1163 .mr = 9,
1164 .qr = 8,
1165 };
1166 xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhanef25c6d2020-07-24 00:59:40 -0700[diff] [blame]1167 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__neon_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1168 .mr = 4,
1169 };
1170 xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhanef25c6d2020-07-24 00:59:40 -0700[diff] [blame]1171 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__neon_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1172 .mr = 9,
1173 };
1174 xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhanef25c6d2020-07-24 00:59:40 -0700[diff] [blame]1175 .mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__neon_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1176 .mr = 9,
1177 .qr = 8,
1178 };
Marat Dukhan660fd192020-03-10 04:55:30 -0700[diff] [blame]1179 xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
1180 .ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__neonfma_c8,
Marat Dukhan69722492019-11-11 19:55:50 -0800[diff] [blame]1181 .pixel_tile = 1,
1182 .channel_tile = 8,
1183 };
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]1184 xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__neon_x8;
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]1185 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__neon_x8;
Marat Dukhan55dde5b2020-07-10 22:48:54 -0700[diff] [blame]1186 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__neon_x16;
Marat Dukhan28813332020-06-10 18:05:38 -0700[diff] [blame]1187 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__neon_x8;
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]1188 xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__neon_x8;
Marat Dukhan64e52512020-06-09 13:41:16 -0700[diff] [blame]1189 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__neonv8_x8;
1190 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__neonv8_x8;
1191 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__neonv8_x8;
1192 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__neonv8_x8;
Marat Dukhan4a24a582020-01-06 13:30:00 -0800[diff] [blame]1193 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__neonfma_rr1_lut64_p2_nr2recps_x16;
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]1194 xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__neon_x8;
Marat Dukhan6804bbd2020-06-30 19:26:11 -0700[diff] [blame]1195 xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__neon_sqrt_x4;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1196 xnn_params.f32.prelu = (struct prelu_parameters) {
Marat Dukhan69c3f2c2019-11-06 12:30:01 -0800[diff] [blame]1197 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__neon_2x8,
1198 .row_tile = 2,
1199 .channel_tile = 8,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1200 };
Marat Dukhan1edc4542020-01-27 12:40:13 -0800[diff] [blame]1201 xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__neonfma_lut64_p2_x16;
1202 xnn_params.f32.rmax = xnn_f32_rmax_ukernel__neon;
Marat Dukhanb1a0fc32019-12-02 19:32:02 -0800[diff] [blame]1203 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1204 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__neon_x8,
1205 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__neon_x8,
1206 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__neon_x8,
Marat Dukhanb1a0fc32019-12-02 19:32:02 -0800[diff] [blame]1207 .element_tile = 8,
1208 };
Marat Dukhan69180502019-12-06 15:00:31 -0800[diff] [blame]1209 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1210 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__neon_x8,
1211 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__neon_x8,
1212 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__neon_x8,
Marat Dukhan69180502019-12-06 15:00:31 -0800[diff] [blame]1213 .element_tile = 8,
1214 };
Marat Dukhan79e7f842019-12-05 14:35:50 -0800[diff] [blame]1215 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1216 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__neon_x8,
1217 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__neon_x8,
1218 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__neon_x8,
Marat Dukhan79e7f842019-12-05 14:35:50 -0800[diff] [blame]1219 .element_tile = 8,
1220 };
1221 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1222 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__neon_x8,
1223 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__neon_x8,
1224 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__neon_x8,
Marat Dukhan79e7f842019-12-05 14:35:50 -0800[diff] [blame]1225 .element_tile = 8,
1226 };
Marat Dukhan1e782c42019-11-21 17:02:40 -0800[diff] [blame]1227 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1228 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__neon_x8,
1229 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__neon_x8,
1230 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__neon_x8,
Marat Dukhanca2733c2019-11-15 23:21:17 -0800[diff] [blame]1231 .element_tile = 8,
1232 };
Marat Dukhan05f3f6d2019-12-03 15:13:53 -0800[diff] [blame]1233 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1234 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__neon_x8,
1235 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__neon_x8,
1236 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__neon_x8,
Marat Dukhan05f3f6d2019-12-03 15:13:53 -0800[diff] [blame]1237 .element_tile = 8,
1238 };
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]1239 xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1240 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__neon_x8,
1241 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__neon_x8,
1242 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__neon_x8,
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]1243 .element_tile = 8,
1244 };
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1245 xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan9531e9f2020-07-24 15:25:02 -0700[diff] [blame]1246 .ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__neonfma_2x,
Marat Dukhan49e6ee92019-11-06 15:55:29 -0800[diff] [blame]1247 .channel_tile = 4,
1248 .row_tile = 2,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1249 };
Marat Dukhanefc47b82019-11-18 09:25:38 -0800[diff] [blame]1250 #ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1251 xnn_params.f32.spmm = (struct spmm_parameters) {
Marat Dukhan355ab432020-04-09 19:01:52 -0700[diff] [blame]1252 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_16x1__neonfma_pipelined,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1253 .mr = 16,
1254 .nr = 1,
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1255 };
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1256 xnn_params.f32.spmm2 = (struct spmm_parameters) {
Marat Dukhan355ab432020-04-09 19:01:52 -0700[diff] [blame]1257 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_16x2__neonfma,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1258 .mr = 16,
1259 .nr = 2,
1260 };
1261 xnn_params.f32.spmm4 = (struct spmm_parameters) {
Marat Dukhan355ab432020-04-09 19:01:52 -0700[diff] [blame]1262 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_16x4__neonfma,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1263 .mr = 16,
1264 .nr = 4,
1265 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1266 xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1267 .ukernel_with_symm_padding =
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1268 (xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__neonfma_2x2,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1269 .output_channel_tile = 4,
1270 .output_height_tile = 2,
1271 .output_width_tile = 2,
1272 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1273 xnn_params.f32.dwconv_chw_3x3 = (struct dwconv_chw_parameters) {
Marat Dukhan1c6cad92020-10-21 15:48:21 -0700[diff] [blame]1274 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3p1__neonfma_3x4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1275 .input_width_tile = 4,
1276 .output_width_tile = 4,
1277 .output_height_tile = 3,
1278 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1279 xnn_params.f32.dwconv_chw_3x3s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]1280 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3s2p1__neonfma_1x4_acc3,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1281 .input_width_tile = 4,
1282 .output_width_tile = 4,
1283 .output_height_tile = 1,
1284 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1285 xnn_params.f32.dwconv_chw_5x5 = (struct dwconv_chw_parameters) {
Marat Dukhan1c6cad92020-10-21 15:48:21 -0700[diff] [blame]1286 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_5x5p2__neonfma_3x4,
Marat Dukhana99918a2019-11-15 14:40:12 -0800[diff] [blame]1287 .input_width_tile = 4,
1288 .output_width_tile = 4,
Erich Elsen4ad51152019-11-19 13:11:53 -0800[diff] [blame]1289 .output_height_tile = 3,
Marat Dukhana99918a2019-11-15 14:40:12 -0800[diff] [blame]1290 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1291 xnn_params.f32.dwconv_chw_5x5s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]1292 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_5x5s2p2__neonfma_1x4_acc2,
Marat Dukhana99918a2019-11-15 14:40:12 -0800[diff] [blame]1293 .input_width_tile = 4,
1294 .output_width_tile = 4,
1295 .output_height_tile = 1,
1296 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1297 xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
1298 .ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__neon_x4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1299 .channel_tile = 4,
1300 };
Marat Dukhanefc47b82019-11-18 09:25:38 -0800[diff] [blame]1301 #endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1302 #endif // XNN_NO_F32_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1303
1304 /**************************** X32 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1305 #ifndef XNN_NO_X32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]1306 init_flags |= XNN_INIT_FLAG_X32;
1307
Marat Dukhan4662b192020-05-21 15:52:03 -0700[diff] [blame]1308 xnn_params.x32.fill = (struct fill_parameters) {
1309 .ukernel = (xnn_fill_ukernel_function) xnn_x32_fill_ukernel__neon,
1310 .row_tile = 1,
1311 };
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1312 xnn_params.x32.pad = (struct pad_parameters) {
Marat Dukhan63523d42020-05-22 17:07:33 -0700[diff] [blame]1313 .ukernel = (xnn_pad_ukernel_function) xnn_x32_pad_ukernel__neon,
1314 .row_tile = 1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1315 };
Marat Dukhan57dccd82020-04-14 00:53:10 -0700[diff] [blame]1316 xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__neon;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1317 xnn_params.x32.zip = (struct zip_parameters) {
1318 .x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__neon,
1319 .x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__neon,
1320 .x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__neon,
1321 .xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__neon,
1322 };
1323 #endif // XNN_NO_X32_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1324
Marat Dukhan1dadbf72019-10-01 10:46:20 -0700[diff] [blame]1325#elif XNN_ARCH_X86 || XNN_ARCH_X86_64
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1326 if (!cpuinfo_has_x86_sse2()) {
1327 xnn_log_error("XNNPACK initialization failed: SSE2 is not supported");
1328 return;
1329 }
1330
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1331 /**************************** QS8 micro-kernels ****************************/
1332 #ifndef XNN_NO_QS8_OPERATORS
1333 init_flags |= XNN_INIT_FLAG_QS8;
1334
Marat Dukhanbb00b1d2020-08-10 11:37:23 -0700[diff] [blame]1335 if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
1336 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_4x16c8__avx512skx);
1337 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_4x16c8__avx512skx);
1338 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x16c8__avx512skx);
1339 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x16c8__avx512skx);
1340 xnn_params.qs8.gemm.mr = 4;
1341 xnn_params.qs8.gemm.nr = 16;
1342 xnn_params.qs8.gemm.log2_kr = 3;
1343 } else if (cpuinfo_has_x86_xop()) {
Marat Dukhan75215d82020-08-07 23:08:03 -0700[diff] [blame]1344 // XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
1345 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_2x4c8__xop_ld64);
1346 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_2x4c8__xop_ld64);
1347 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x4c8__xop_ld64);
1348 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x4c8__xop_ld64);
1349 xnn_params.qs8.gemm.mr = 2;
1350 xnn_params.qs8.gemm.nr = 4;
1351 xnn_params.qs8.gemm.log2_kr = 3;
1352 } else if (cpuinfo_has_x86_avx2()) {
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1353 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_3x8c8__avx2);
1354 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_3x8c8__avx2);
1355 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x8c8__avx2);
1356 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x8c8__avx2);
1357 xnn_params.qs8.gemm.mr = 3;
1358 xnn_params.qs8.gemm.nr = 8;
1359 xnn_params.qs8.gemm.log2_kr = 3;
1360 } else if (cpuinfo_has_x86_sse4_1()) {
1361 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_3x4c8__sse41_ld64);
1362 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_3x4c8__sse41_ld64);
1363 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x4c8__sse41_ld64);
1364 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x4c8__sse41_ld64);
1365 xnn_params.qs8.gemm.mr = 3;
1366 xnn_params.qs8.gemm.nr = 4;
1367 xnn_params.qs8.gemm.log2_kr = 3;
1368 } else if (cpuinfo_has_x86_ssse3()) {
1369 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_3x4c8__ssse3_ld64);
1370 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_3x4c8__ssse3_ld64);
1371 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x4c8__ssse3_ld64);
1372 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x4c8__ssse3_ld64);
1373 xnn_params.qs8.gemm.mr = 3;
1374 xnn_params.qs8.gemm.nr = 4;
1375 xnn_params.qs8.gemm.log2_kr = 3;
1376 } else {
Marat Dukhan07e50402020-08-05 17:16:53 -0700[diff] [blame]1377 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_3x4c8__sse2_ld64);
1378 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_3x4c8__sse2_ld64);
1379 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x4c8__sse2_ld64);
1380 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x4c8__sse2_ld64);
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1381 xnn_params.qs8.gemm.mr = 3;
1382 xnn_params.qs8.gemm.nr = 4;
1383 xnn_params.qs8.gemm.log2_kr = 3;
1384 }
1385
Marat Dukhan2ffc5e62020-09-06 22:33:38 -0700[diff] [blame]1386 if (cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl()) {
1387 xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up32x9__avx512skx_mul32;
1388 xnn_params.qs8.dwconv[0].channel_tile = 32;
1389 } else if (cpuinfo_has_x86_avx2()) {
Marat Dukhand65a1522020-08-04 19:28:18 -0700[diff] [blame]1390 xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up16x9__avx2_mul32;
1391 xnn_params.qs8.dwconv[0].channel_tile = 16;
1392 } else if (cpuinfo_has_x86_sse4_1()) {
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1393 xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x9__sse41_mul16;
Marat Dukhand65a1522020-08-04 19:28:18 -0700[diff] [blame]1394 xnn_params.qs8.dwconv[0].channel_tile = 8;
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1395 } else if (cpuinfo_has_x86_ssse3()) {
1396 xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x9__ssse3_mul16;
Marat Dukhand65a1522020-08-04 19:28:18 -0700[diff] [blame]1397 xnn_params.qs8.dwconv[0].channel_tile = 8;
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1398 } else if (cpuinfo_has_x86_sse2()) {
1399 xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x9__sse2_mul16;
Marat Dukhand65a1522020-08-04 19:28:18 -0700[diff] [blame]1400 xnn_params.qs8.dwconv[0].channel_tile = 8;
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1401 }
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1402 xnn_params.qs8.dwconv[0].primary_tile = 9;
Marat Dukhan9e0b5392020-08-07 02:29:34 -0700[diff] [blame]1403
1404 if (cpuinfo_has_x86_sse4_1()) {
1405 xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
1406 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__sse41_c8_acc2,
1407 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__sse41_c8_acc2,
1408 .mr = 7,
1409 };
1410 } else if (cpuinfo_has_x86_ssse3()) {
1411 xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
1412 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__ssse3_c8_acc2,
1413 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__ssse3_c8_acc2,
1414 .mr = 7,
1415 };
1416 } else if (cpuinfo_has_x86_sse2()) {
1417 xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
1418 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__sse2_c8_acc2,
1419 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__sse2_c8_acc2,
1420 .mr = 7,
1421 };
1422 }
Marat Dukhanff209482020-09-03 14:26:53 -0700[diff] [blame]1423
Marat Dukhanbb9225e2020-09-06 22:40:56 -0700[diff] [blame]1424 if (cpuinfo_has_x86_xop()) {
1425 xnn_params.qs8.vadd = (struct vbinary_parameters) {
1426 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__xop_mul32_ld32_x8,
1427 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__xop_mul32_ld32_x8,
1428 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__xop_mul32_ld32_x8,
1429 .element_tile = 8,
1430 };
1431 } else if (cpuinfo_has_x86_sse4_1()) {
Marat Dukhanff209482020-09-03 14:26:53 -0700[diff] [blame]1432 xnn_params.qs8.vadd = (struct vbinary_parameters) {
1433 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__sse41_mul16_ld64_x8,
1434 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__sse41_mul16_ld64_x8,
1435 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__sse41_mul16_ld64_x8,
1436 .element_tile = 8,
1437 };
1438 } else {
1439 xnn_params.qs8.vadd = (struct vbinary_parameters) {
1440 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__sse2_mul16_ld64_x8,
1441 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__sse2_mul16_ld64_x8,
1442 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__sse2_mul16_ld64_x8,
1443 .element_tile = 8,
1444 };
1445 }
Marat Dukhan07e50402020-08-05 17:16:53 -0700[diff] [blame]1446 #endif // XNN_NO_QS8_OPERATORS
Marat Dukhan16f1e1a2020-08-04 16:38:22 -0700[diff] [blame]1447
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1448 /**************************** QU8 micro-kernels ****************************/
1449 #ifndef XNN_NO_QU8_OPERATORS
1450 init_flags |= XNN_INIT_FLAG_QU8;
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]1451
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1452 xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_ukernel_4x4c2__sse2);
1453 xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_ukernel_4x4c2__sse2);
1454 xnn_params.qu8.gemm.mr = 4;
1455 xnn_params.qu8.gemm.nr = 4;
1456 xnn_params.qu8.gemm.log2_kr = 1;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1457
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1458 xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_ukernel_up8x9__sse2;
1459 xnn_params.qu8.dwconv[0].channel_tile = 8;
1460 xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1461
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1462 xnn_params.qu8.avgpool = (struct avgpool_parameters) {
1463 .up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__sse2_c8,
1464 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__sse2_c8,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1465 .mr = 9,
1466 .qr = 8,
1467 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1468 xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
1469 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__sse2_c8,
1470 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__sse2_c8,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1471 .mr = 7,
1472 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1473 xnn_params.qu8.vadd = (xnn_vadd_ukernel_function) xnn_qu8_vadd_minmax_ukernel__sse2;
1474 #endif // XNN_NO_QU8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1475
1476 /**************************** U8 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1477 #ifndef XNN_NO_U8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]1478 init_flags |= XNN_INIT_FLAG_U8;
1479
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1480 xnn_params.u8.maxpool = (struct maxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1481 .ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__sse2_c16,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1482 .mr = 9,
1483 .qr = 8,
1484 };
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]1485 xnn_params.u8.clamp = (xnn_univector_ukernel_function) xnn_u8_clamp_ukernel__sse2_x64;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1486 xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
1487 xnn_params.u8.rmax = xnn_u8_rmax_ukernel__sse2;
1488 #endif // XNN_NO_U8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1489
1490 /**************************** X8 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1491 #ifndef XNN_NO_X8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]1492 init_flags |= XNN_INIT_FLAG_X8;
1493
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1494 xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar;
1495 xnn_params.x8.zip = (struct zip_parameters) {
1496 .x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__sse2,
1497 .x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__sse2,
1498 .x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__sse2,
1499 .xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__sse2,
1500 };
1501 #endif // XNN_NO_X8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1502
1503 /**************************** F32 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1504 #ifndef XNN_NO_F32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]1505 init_flags |= XNN_INIT_FLAG_F32;
1506
Marat Dukhan0f349c42019-11-27 11:58:54 -0800[diff] [blame]1507 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1508 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_7x16__avx512f_broadcast);
1509 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_7x16__avx512f_broadcast);
1510 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x16__avx512f_broadcast);
1511 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x16__avx512f_broadcast);
1512 xnn_params.f32.gemm.mr = 7;
1513 xnn_params.f32.gemm.nr = 16;
Marat Dukhan0f349c42019-11-27 11:58:54 -0800[diff] [blame]1514 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_fma3()) {
Marat Dukhan27121322019-12-09 14:57:40 -0800[diff] [blame]1515 switch (cpuinfo_get_core(0)->uarch) {
1516 case cpuinfo_uarch_zen:
Marat Dukhanb3801eb2020-03-12 13:41:11 -0700[diff] [blame]1517 case cpuinfo_uarch_dhyana:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1518 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x16s4__fma3_broadcast);
1519 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x16s4__fma3_broadcast);
1520 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x16s4__fma3_broadcast);
1521 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x16s4__fma3_broadcast);
1522 xnn_params.f32.gemm.mr = 4;
1523 xnn_params.f32.gemm.nr = 16;
1524 xnn_params.f32.gemm.log2_sr = 2;
Marat Dukhan27121322019-12-09 14:57:40 -0800[diff] [blame]1525 break;
1526 default:
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1527 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_5x16__fma3_broadcast);
1528 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_5x16__fma3_broadcast);
1529 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x16__fma3_broadcast);
1530 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x16__fma3_broadcast);
1531 xnn_params.f32.gemm.mr = 5;
1532 xnn_params.f32.gemm.nr = 16;
Marat Dukhan27121322019-12-09 14:57:40 -0800[diff] [blame]1533 break;
1534 }
Marat Dukhan1025ea32019-11-21 16:01:08 -0800[diff] [blame]1535 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1536 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_5x16__avx_broadcast);
1537 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_5x16__avx_broadcast);
1538 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x16__avx_broadcast);
1539 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x16__avx_broadcast);
1540 xnn_params.f32.gemm.mr = 5;
1541 xnn_params.f32.gemm.nr = 16;
Marat Dukhan1025ea32019-11-21 16:01:08 -0800[diff] [blame]1542 } else {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1543 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__sse_load1);
1544 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__sse_load1);
1545 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__sse_load1);
1546 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__sse_load1);
1547 xnn_params.f32.gemm.mr = 4;
1548 xnn_params.f32.gemm.nr = 8;
Marat Dukhan1025ea32019-11-21 16:01:08 -0800[diff] [blame]1549 }
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1550 xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2c4__sse);
1551 xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2c4__sse);
1552 xnn_params.f32.gemm2.mr = 4;
1553 xnn_params.f32.gemm2.nr = 2;
1554 xnn_params.f32.gemm2.log2_kr = 2;
1555
Marat Dukhan479f87e2019-11-27 15:17:06 -0800[diff] [blame]1556 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1557 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x4__avx512f;
1558 xnn_params.f32.dwconv[0].channel_tile = 16;
1559 xnn_params.f32.dwconv[0].primary_tile = 4;
1560
1561 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x9__avx512f;
1562 xnn_params.f32.dwconv[1].channel_tile = 16;
1563 xnn_params.f32.dwconv[1].primary_tile = 9;
1564
1565 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x25__avx512f;
1566 xnn_params.f32.dwconv[2].channel_tile = 16;
1567 xnn_params.f32.dwconv[2].primary_tile = 25;
Marat Dukhan479f87e2019-11-27 15:17:06 -0800[diff] [blame]1568 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_fma3()) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1569 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x4__fma3;
1570 xnn_params.f32.dwconv[0].channel_tile = 16;
1571 xnn_params.f32.dwconv[0].primary_tile = 4;
1572
1573 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x9__fma3;
1574 xnn_params.f32.dwconv[1].channel_tile = 16;
1575 xnn_params.f32.dwconv[1].primary_tile = 9;
1576
1577 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x25__fma3;
1578 xnn_params.f32.dwconv[2].channel_tile = 8;
1579 xnn_params.f32.dwconv[2].primary_tile = 25;
Marat Dukhan17ec5f32019-11-22 13:34:16 -0800[diff] [blame]1580 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1581 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x4__avx;
1582 xnn_params.f32.dwconv[0].channel_tile = 16;
1583 xnn_params.f32.dwconv[0].primary_tile = 4;
1584
1585 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up16x9__avx;
1586 xnn_params.f32.dwconv[1].channel_tile = 16;
1587 xnn_params.f32.dwconv[1].primary_tile = 9;
1588
1589 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x25__avx;
1590 xnn_params.f32.dwconv[2].channel_tile = 8;
1591 xnn_params.f32.dwconv[2].primary_tile = 25;
Marat Dukhan17ec5f32019-11-22 13:34:16 -0800[diff] [blame]1592 } else {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1593 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x4__sse;
1594 xnn_params.f32.dwconv[0].channel_tile = 8;
1595 xnn_params.f32.dwconv[0].primary_tile = 4;
1596
1597 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__sse;
1598 xnn_params.f32.dwconv[1].channel_tile = 8;
1599 xnn_params.f32.dwconv[1].primary_tile = 9;
1600
1601 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x25__sse;
1602 xnn_params.f32.dwconv[2].channel_tile = 8;
1603 xnn_params.f32.dwconv[2].primary_tile = 25;
Marat Dukhan17ec5f32019-11-22 13:34:16 -0800[diff] [blame]1604 }
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1605 xnn_params.f32.avgpool = (struct avgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1606 .up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__sse_c4,
1607 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__sse_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1608 .mr = 9,
1609 .qr = 8,
1610 };
1611 xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1612 .up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__sse_c4,
1613 .mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__sse_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1614 .mr = 9,
1615 .qr = 8,
1616 };
1617 xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1618 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__sse_c4,
1619 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__sse_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1620 .mr = 7,
1621 };
1622 xnn_params.f32.maxpool = (struct maxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1623 .ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__sse_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1624 .mr = 9,
1625 .qr = 8,
1626 };
1627 xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1628 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__sse2_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1629 .mr = 4,
1630 };
1631 xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1632 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__sse2_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1633 .mr = 9,
1634 };
1635 xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]1636 .mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__sse2_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1637 .mr = 9,
1638 .qr = 8,
1639 };
Marat Dukhan660fd192020-03-10 04:55:30 -0700[diff] [blame]1640 xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
1641 .ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__sse_c8,
Marat Dukhan69722492019-11-11 19:55:50 -0800[diff] [blame]1642 .pixel_tile = 1,
1643 .channel_tile = 8,
1644 };
Marat Dukhane2c3f292019-11-27 15:40:54 -0800[diff] [blame]1645 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]1646 xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__avx512f_x16;
1647 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1648 xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__avx_x16;
1649 } else {
1650 xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__sse_x8;
1651 }
1652 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]1653 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__avx512f_x16;
Marat Dukhane2c3f292019-11-27 15:40:54 -0800[diff] [blame]1654 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]1655 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__avx_x16;
Marat Dukhane2c3f292019-11-27 15:40:54 -0800[diff] [blame]1656 } else {
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]1657 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__sse_x8;
Marat Dukhane2c3f292019-11-27 15:40:54 -0800[diff] [blame]1658 }
Marat Dukhan662faa02019-12-09 22:48:16 -0800[diff] [blame]1659 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]1660 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__avx512f_x16;
Marat Dukhan662faa02019-12-09 22:48:16 -0800[diff] [blame]1661 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_fma3()) {
1662 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__fma3_x16;
1663 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1664 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__avx_x16;
1665 } else {
1666 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__sse_x8;
1667 }
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]1668 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan28813332020-06-10 18:05:38 -0700[diff] [blame]1669 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__avx512f_x16;
1670 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1671 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__avx_x16;
Marat Dukhan0d3f4672020-06-25 16:42:58 -0700[diff] [blame]1672 } else if (cpuinfo_has_x86_sse4_1()) {
1673 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__sse41_x8;
Marat Dukhan28813332020-06-10 18:05:38 -0700[diff] [blame]1674 } else {
1675 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__sse_x8;
1676 }
1677 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]1678 xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__avx512f_x16;
1679 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1680 xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__avx_x16;
1681 } else {
1682 xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__sse_x8;
1683 }
Marat Dukhan64e52512020-06-09 13:41:16 -0700[diff] [blame]1684 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
1685 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__avx512f_x16;
1686 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__avx512f_x16;
1687 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__avx512f_x16;
1688 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__avx512f_x16;
1689 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1690 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__avx_x16;
1691 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__avx_x16;
1692 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__avx_x16;
1693 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__avx_x16;
1694 } else if (cpuinfo_has_x86_sse4_1()) {
1695 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__sse41_x8;
1696 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__sse41_x8;
1697 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__sse41_x8;
1698 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__sse41_x8;
1699 } else {
1700 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__sse2_x8;
1701 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__sse2_x8;
1702 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__sse2_x8;
1703 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__sse2_x8;
1704 }
Marat Dukhand9ca7e62020-09-23 23:45:29 -0700[diff] [blame]1705 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
1706 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__avx512f_rr2_lut32_p2_perm2_scalef_div_x64;
1707 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx2()) {
Marat Dukhanfa0a4322020-01-06 16:14:29 -0800[diff] [blame]1708 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__avx2_rr1_p5_div_x40;
T.J. Alumbaughdc2b29c2020-10-14 13:56:08 -0700[diff] [blame]1709 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1710 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__avx_rr2_p5_nr2_x40;
Marat Dukhan6dd71362020-09-17 23:11:21 -0700[diff] [blame]1711 } else if (cpuinfo_has_x86_sse4_1()) {
1712 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__sse41_lut64_p2_div_x8;
Marat Dukhanfa0a4322020-01-06 16:14:29 -0800[diff] [blame]1713 } else {
Marat Dukhan6dd71362020-09-17 23:11:21 -0700[diff] [blame]1714 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__sse2_lut64_p2_div_x8;
Marat Dukhanfa0a4322020-01-06 16:14:29 -0800[diff] [blame]1715 }
Marat Dukhan90eca0a2020-03-11 00:52:23 -0700[diff] [blame]1716 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]1717 xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__avx512f_x16;
1718 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1719 xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__avx_x16;
1720 } else {
1721 xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__sse_x8;
1722 }
Marat Dukhan6804bbd2020-06-30 19:26:11 -0700[diff] [blame]1723 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1724 xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__avx_sqrt_x8;
1725 } else {
1726 xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__sse_sqrt_x4;
1727 }
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]1728 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
Marat Dukhan90eca0a2020-03-11 00:52:23 -0700[diff] [blame]1729 xnn_params.f32.prelu = (struct prelu_parameters) {
1730 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__avx512f_2x16,
1731 .row_tile = 2,
1732 .channel_tile = 16,
1733 };
1734 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1735 xnn_params.f32.prelu = (struct prelu_parameters) {
1736 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__avx_2x16,
1737 .row_tile = 2,
1738 .channel_tile = 16,
1739 };
Marat Dukhan39b5e942020-06-24 15:03:48 -0700[diff] [blame]1740 } else if (cpuinfo_has_x86_sse4_1()) {
1741 xnn_params.f32.prelu = (struct prelu_parameters) {
1742 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__sse41_2x8,
1743 .row_tile = 2,
1744 .channel_tile = 8,
1745 };
Marat Dukhan90eca0a2020-03-11 00:52:23 -0700[diff] [blame]1746 } else {
1747 xnn_params.f32.prelu = (struct prelu_parameters) {
1748 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__sse2_2x8,
1749 .row_tile = 2,
1750 .channel_tile = 8,
1751 };
1752 }
Marat Dukhan1edc4542020-01-27 12:40:13 -0800[diff] [blame]1753 xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__sse2_p5_x20_acc2;
1754 xnn_params.f32.rmax = xnn_f32_rmax_ukernel__sse;
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1755 if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx512f()) {
1756 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1757 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__avx512f_x32,
1758 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__avx512f_x32,
1759 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__avx512f_x32,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1760 .element_tile = 32,
1761 };
1762 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1763 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__avx512f_x32,
1764 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__avx512f_x32,
1765 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__avx512f_x32,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1766 .element_tile = 32,
1767 };
1768 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1769 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__avx512f_x32,
1770 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__avx512f_x32,
1771 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__avx512f_x32,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1772 .element_tile = 32,
1773 };
1774 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1775 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__avx512f_x32,
1776 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__avx512f_x32,
1777 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__avx512f_x32,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1778 .element_tile = 32,
1779 };
1780 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1781 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__avx512f_x32,
1782 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__avx512f_x32,
1783 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__avx512f_x32,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1784 .element_tile = 32,
1785 };
1786 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1787 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__avx512f_x32,
1788 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__avx512f_x32,
1789 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__avx512f_x32,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1790 .element_tile = 32,
1791 };
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]1792 xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1793 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__avx512f_x32,
1794 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__avx512f_x32,
1795 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__avx512f_x32,
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]1796 .element_tile = 32,
1797 };
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1798 } else if (!XNN_PLATFORM_MOBILE && cpuinfo_has_x86_avx()) {
1799 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1800 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__avx_x16,
1801 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__avx_x16,
1802 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__avx_x16,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1803 .element_tile = 16,
1804 };
1805 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1806 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__avx_x16,
1807 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__avx_x16,
1808 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__avx_x16,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1809 .element_tile = 16,
1810 };
1811 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1812 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__avx_x16,
1813 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__avx_x16,
1814 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__avx_x16,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1815 .element_tile = 16,
1816 };
1817 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1818 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__avx_x16,
1819 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__avx_x16,
1820 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__avx_x16,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1821 .element_tile = 16,
1822 };
1823 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1824 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__avx_x16,
1825 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__avx_x16,
1826 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__avx_x16,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1827 .element_tile = 16,
1828 };
1829 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1830 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__avx_x16,
1831 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__avx_x16,
1832 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__avx_x16,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1833 .element_tile = 16,
1834 };
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]1835 xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1836 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__avx_x16,
1837 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__avx_x16,
1838 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__avx_x16,
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]1839 .element_tile = 16,
1840 };
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1841 } else {
1842 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1843 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__sse_x8,
1844 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__sse_x8,
1845 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__sse_x8,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1846 .element_tile = 8,
1847 };
1848 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1849 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__sse_x8,
1850 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__sse_x8,
1851 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__sse_x8,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1852 .element_tile = 8,
1853 };
1854 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1855 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__sse_x8,
1856 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__sse_x8,
1857 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__sse_x8,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1858 .element_tile = 8,
1859 };
1860 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1861 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__sse_x8,
1862 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__sse_x8,
1863 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__sse_x8,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1864 .element_tile = 8,
1865 };
1866 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1867 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__sse_x8,
1868 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__sse_x8,
1869 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__sse_x8,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1870 .element_tile = 8,
1871 };
1872 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1873 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__sse_x8,
1874 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__sse_x8,
1875 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__sse_x8,
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1876 .element_tile = 8,
1877 };
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]1878 xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]1879 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__sse_x8,
1880 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__sse_x8,
1881 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__sse_x8,
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]1882 .element_tile = 8,
1883 };
Marat Dukhan9a88efe2019-12-10 15:54:24 -0800[diff] [blame]1884 }
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1885 xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan9531e9f2020-07-24 15:25:02 -0700[diff] [blame]1886 .ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__sse_2x,
Marat Dukhan49e6ee92019-11-06 15:55:29 -0800[diff] [blame]1887 .channel_tile = 4,
1888 .row_tile = 2,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1889 };
Marat Dukhanefc47b82019-11-18 09:25:38 -0800[diff] [blame]1890 #ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1891 xnn_params.f32.spmm = (struct spmm_parameters) {
Erich Elsen6e80fdc2020-06-09 15:35:37 -0700[diff] [blame]1892 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_16x1__sse,
1893 .mr = 16,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1894 .nr = 1,
1895 };
Erich Elsen5b2e07a2020-06-09 03:27:59 -0700[diff] [blame]1896 xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
1897 .ukernel_with_symm_padding =
1898 (xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__sse_2x2,
1899 .output_channel_tile = 4,
1900 .output_height_tile = 2,
1901 .output_width_tile = 2,
1902 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1903 xnn_params.f32.dwconv_chw_3x3 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]1904 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3p1__sse_1x4_acc3,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1905 .input_width_tile = 4,
1906 .output_width_tile = 4,
1907 .output_height_tile = 1,
1908 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1909 xnn_params.f32.dwconv_chw_3x3s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]1910 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3s2p1__sse_1x4_acc3,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1911 .input_width_tile = 4,
1912 .output_width_tile = 4,
1913 .output_height_tile = 1,
1914 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]1915 xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
1916 .ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__sse_x4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1917 .channel_tile = 4,
1918 };
Marat Dukhanefc47b82019-11-18 09:25:38 -0800[diff] [blame]1919 #endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1920 #endif // XNN_NO_F32_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1921
1922 /**************************** X32 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1923 #ifndef XNN_NO_X32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]1924 init_flags |= XNN_INIT_FLAG_X32;
1925
Marat Dukhan4662b192020-05-21 15:52:03 -0700[diff] [blame]1926 xnn_params.x32.fill = (struct fill_parameters) {
1927 .ukernel = (xnn_fill_ukernel_function) xnn_x32_fill_ukernel__sse,
1928 .row_tile = 1,
1929 };
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1930 xnn_params.x32.pad = (struct pad_parameters) {
Marat Dukhan63523d42020-05-22 17:07:33 -0700[diff] [blame]1931 .ukernel = (xnn_pad_ukernel_function) xnn_x32_pad_ukernel__sse,
1932 .row_tile = 1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1933 };
Marat Dukhan57dccd82020-04-14 00:53:10 -0700[diff] [blame]1934 xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__sse2;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1935 xnn_params.x32.zip = (struct zip_parameters) {
1936 .x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__sse2,
1937 .x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__sse2,
1938 .x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__sse2,
1939 .xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__sse2,
1940 };
1941 #endif // XNN_NO_X32_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]1942
Marat Dukhanf42facc2020-03-08 15:14:53 -0700[diff] [blame]1943#elif XNN_ARCH_WASMSIMD
Marat Dukhan07e50402020-08-05 17:16:53 -0700[diff] [blame]1944 /**************************** QS8 micro-kernels ****************************/
1945 #ifndef XNN_NO_QS8_OPERATORS
1946 init_flags |= XNN_INIT_FLAG_QS8;
1947
1948 xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_3x4c8__wasmsimd_ld64);
1949 xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_3x4c8__wasmsimd_ld64);
1950 xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x4c8__wasmsimd_ld64);
1951 xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x4c8__wasmsimd_ld64);
1952 xnn_params.qs8.gemm.mr = 3;
1953 xnn_params.qs8.gemm.nr = 4;
1954 xnn_params.qs8.gemm.log2_kr = 3;
1955
1956 xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x9__wasmsimd_mul16;
1957 xnn_params.qs8.dwconv[0].channel_tile = 8;
1958 xnn_params.qs8.dwconv[0].primary_tile = 9;
Marat Dukhan9e0b5392020-08-07 02:29:34 -0700[diff] [blame]1959
1960 xnn_params.qs8.gavgpool = (struct gavgpool_parameters) {
1961 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7x__wasmsimd_c8_acc2,
1962 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qs8_gavgpool_minmax_ukernel_7p7x__wasmsimd_c8_acc2,
1963 .mr = 7,
1964 };
Marat Dukhanff209482020-09-03 14:26:53 -0700[diff] [blame]1965
1966 xnn_params.qs8.vadd = (struct vbinary_parameters) {
1967 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vadd_minmax_ukernel__wasmsimd_x8,
1968 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__wasmsimd_x8,
1969 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_qs8_vaddc_minmax_ukernel__wasmsimd_x8,
1970 .element_tile = 8,
1971 };
Marat Dukhan07e50402020-08-05 17:16:53 -0700[diff] [blame]1972 #endif // XNN_NO_QS8_OPERATORS
1973
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1974 /**************************** QU8 micro-kernels ****************************/
1975 #ifndef XNN_NO_QU8_OPERATORS
1976 init_flags |= XNN_INIT_FLAG_QU8;
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]1977
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1978 xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_ukernel_2x2__scalar);
1979 xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_ukernel_2x2__scalar);
1980 xnn_params.qu8.gemm.mr = 2;
1981 xnn_params.qu8.gemm.nr = 2;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1982
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1983 xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_ukernel_up1x9__scalar;
1984 xnn_params.qu8.dwconv[0].channel_tile = 1;
1985 xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]1986
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1987 xnn_params.qu8.avgpool = (struct avgpool_parameters) {
1988 .up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__scalar_c1,
1989 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1990 .mr = 9,
1991 .qr = 8,
1992 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1993 xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
1994 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__scalar_c1,
1995 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]1996 .mr = 7,
1997 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]1998 xnn_params.qu8.vadd = (xnn_vadd_ukernel_function) xnn_qu8_vadd_minmax_ukernel__scalar;
1999 #endif // XNN_NO_QU8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2000
2001 /**************************** U8 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2002 #ifndef XNN_NO_U8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2003 init_flags |= XNN_INIT_FLAG_U8;
2004
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2005 xnn_params.u8.maxpool = (struct maxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2006 .ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2007 .mr = 9,
2008 .qr = 8,
2009 };
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]2010 xnn_params.u8.clamp = (xnn_univector_ukernel_function) xnn_u8_clamp_ukernel__scalar_x4;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2011 xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
2012 xnn_params.u8.rmax = xnn_u8_rmax_ukernel__scalar;
2013 #endif // XNN_NO_U8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2014
2015 /**************************** X8 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2016 #ifndef XNN_NO_X8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2017 init_flags |= XNN_INIT_FLAG_X8;
2018
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2019 xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar;
2020 xnn_params.x8.zip = (struct zip_parameters) {
2021 .x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__scalar,
2022 .x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__scalar,
2023 .x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__scalar,
2024 .xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__scalar,
2025 };
2026 #endif // XNN_NO_X8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2027
2028 /**************************** F32 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2029 #ifndef XNN_NO_F32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2030 init_flags |= XNN_INIT_FLAG_F32;
2031
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2032 if (is_wasm_x86) {
Marat Dukhan1bbf96b2020-06-15 23:01:20 -0700[diff] [blame]2033 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_splat_x86);
2034 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_splat_x86);
2035 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_splat_x86);
2036 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__wasmsimd_splat_x86);
Marat Dukhan688f6d82020-07-14 17:02:11 -0700[diff] [blame]2037 xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_4x8__wasmsimd_splat);
2038 xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_4x8__wasmsimd_splat);
2039 xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x8__wasmsimd_splat);
2040 xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x8__wasmsimd_splat);
Marat Dukhan802808c2020-06-16 11:01:17 -0700[diff] [blame]2041 xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x8__wasmsimd_splat);
2042 xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x8__wasmsimd_splat);
2043 xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x8__wasmsimd_splat);
2044 xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x8__wasmsimd_splat);
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2045 xnn_params.f32.gemm.mr = 4;
2046 xnn_params.f32.gemm.nr = 8;
Marat Dukhane39e6462020-07-09 01:33:36 -0700[diff] [blame]2047
2048 xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2c4__wasmsimd_x86);
2049 xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2c4__wasmsimd_x86);
2050 xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2c4__wasmsimd);
2051 xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2c4__wasmsimd);
2052 xnn_params.f32.gemm2.mr = 4;
2053 xnn_params.f32.gemm2.nr = 2;
2054 xnn_params.f32.gemm2.log2_kr = 2;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2055 } else {
Marat Dukhan802808c2020-06-16 11:01:17 -0700[diff] [blame]2056 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_splat_arm);
2057 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_splat_arm);
2058 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_splat_arm);
2059 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x8__wasmsimd_splat_arm);
Marat Dukhan688f6d82020-07-14 17:02:11 -0700[diff] [blame]2060 xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_5x8__wasmsimd_splat);
2061 xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_5x8__wasmsimd_splat);
2062 xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x8__wasmsimd_splat);
2063 xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x8__wasmsimd_splat);
Marat Dukhan802808c2020-06-16 11:01:17 -0700[diff] [blame]2064 xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_5x8__wasmsimd_splat);
2065 xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_5x8__wasmsimd_splat);
2066 xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x8__wasmsimd_splat);
2067 xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x8__wasmsimd_splat);
Marat Dukhan1bbf96b2020-06-15 23:01:20 -0700[diff] [blame]2068 xnn_params.f32.gemm.mr = 5;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2069 xnn_params.f32.gemm.nr = 8;
Marat Dukhane39e6462020-07-09 01:33:36 -0700[diff] [blame]2070
2071 xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2c4__wasmsimd_arm);
2072 xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2c4__wasmsimd_arm);
2073 xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2c4__wasmsimd);
2074 xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2c4__wasmsimd);
2075 xnn_params.f32.gemm2.mr = 4;
2076 xnn_params.f32.gemm2.nr = 2;
2077 xnn_params.f32.gemm2.log2_kr = 2;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2078 }
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2079
Marat Dukhanac014d72020-06-16 08:36:47 -0700[diff] [blame]2080 if (is_wasm_x86) {
2081 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86;
Marat Dukhanb8e7b072020-06-16 12:34:23 -0700[diff] [blame]2082 xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up8x4__wasmsimd;
Marat Dukhanac014d72020-06-16 08:36:47 -0700[diff] [blame]2083 xnn_params.f32.dwconv[0].channel_tile = 8;
2084 xnn_params.f32.dwconv[0].primary_tile = 4;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2085
Marat Dukhanac014d72020-06-16 08:36:47 -0700[diff] [blame]2086 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86;
Marat Dukhanb8e7b072020-06-16 12:34:23 -0700[diff] [blame]2087 xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up8x9__wasmsimd;
Marat Dukhanac014d72020-06-16 08:36:47 -0700[diff] [blame]2088 xnn_params.f32.dwconv[1].channel_tile = 8;
2089 xnn_params.f32.dwconv[1].primary_tile = 9;
2090 } else {
2091 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm;
Marat Dukhanb8e7b072020-06-16 12:34:23 -0700[diff] [blame]2092 xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up4x4__wasmsimd;
Marat Dukhanac014d72020-06-16 08:36:47 -0700[diff] [blame]2093 xnn_params.f32.dwconv[0].channel_tile = 4;
2094 xnn_params.f32.dwconv[0].primary_tile = 4;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2095
Marat Dukhanac014d72020-06-16 08:36:47 -0700[diff] [blame]2096 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm;
Marat Dukhanb8e7b072020-06-16 12:34:23 -0700[diff] [blame]2097 xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up4x9__wasmsimd;
Marat Dukhanac014d72020-06-16 08:36:47 -0700[diff] [blame]2098 xnn_params.f32.dwconv[1].channel_tile = 4;
2099 xnn_params.f32.dwconv[1].primary_tile = 9;
2100 }
2101
2102 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm;
Marat Dukhanb8e7b072020-06-16 12:34:23 -0700[diff] [blame]2103 xnn_params.f32.dwconv[2].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up4x25__wasmsimd;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2104 xnn_params.f32.dwconv[2].channel_tile = 4;
2105 xnn_params.f32.dwconv[2].primary_tile = 25;
2106
Marat Dukhan3b7432d2020-07-16 17:46:32 -0700[diff] [blame]2107 if (is_wasm_x86) {
2108 xnn_params.f32.avgpool = (struct avgpool_parameters) {
2109 .up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__wasmsimd_x86_c4,
2110 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__wasmsimd_x86_c4,
2111 .mr = 9,
2112 .qr = 8,
2113 };
Marat Dukhan1483c532020-07-16 18:08:19 -0700[diff] [blame]2114 xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
2115 .up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__wasmsimd_x86_c4,
2116 .mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__wasmsimd_x86_c4,
2117 .mr = 9,
2118 .qr = 8,
2119 };
Marat Dukhanc6016802020-07-16 18:51:28 -0700[diff] [blame]2120 xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
2121 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__wasmsimd_x86_c4,
2122 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__wasmsimd_x86_c4,
2123 .mr = 7,
2124 };
Marat Dukhan3b7432d2020-07-16 17:46:32 -0700[diff] [blame]2125 } else {
2126 xnn_params.f32.avgpool = (struct avgpool_parameters) {
2127 .up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__wasmsimd_arm_c4,
2128 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__wasmsimd_arm_c4,
2129 .mr = 9,
2130 .qr = 8,
2131 };
Marat Dukhan1483c532020-07-16 18:08:19 -0700[diff] [blame]2132 xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
2133 .up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__wasmsimd_arm_c4,
2134 .mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__wasmsimd_arm_c4,
2135 .mr = 9,
2136 .qr = 8,
2137 };
Marat Dukhanc6016802020-07-16 18:51:28 -0700[diff] [blame]2138 xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
2139 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__wasmsimd_arm_c4,
2140 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__wasmsimd_arm_c4,
2141 .mr = 7,
2142 };
Marat Dukhan3b7432d2020-07-16 17:46:32 -0700[diff] [blame]2143 }
Marat Dukhanf6e24802020-07-08 22:20:40 -0700[diff] [blame]2144 if (is_wasm_x86) {
2145 xnn_params.f32.maxpool = (struct maxpool_parameters) {
2146 .ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__wasmsimd_x86_c4,
2147 .mr = 9,
2148 .qr = 8,
2149 };
2150 } else {
2151 xnn_params.f32.maxpool = (struct maxpool_parameters) {
2152 .ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__wasmsimd_arm_c4,
2153 .mr = 9,
2154 .qr = 8,
2155 };
2156 }
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2157 xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhan40f05522020-07-16 22:33:12 -0700[diff] [blame]2158 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__wasmsimd_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2159 .mr = 4,
2160 };
2161 xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhan40f05522020-07-16 22:33:12 -0700[diff] [blame]2162 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__wasmsimd_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2163 .mr = 9,
2164 };
2165 xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhan40f05522020-07-16 22:33:12 -0700[diff] [blame]2166 .mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__wasmsimd_c4,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2167 .mr = 9,
2168 .qr = 8,
2169 };
Marat Dukhan660fd192020-03-10 04:55:30 -0700[diff] [blame]2170 xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
Marat Dukhan00d1d6e2020-07-09 01:37:27 -0700[diff] [blame]2171 .ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__wasmsimd_c8,
Marat Dukhan69722492019-11-11 19:55:50 -0800[diff] [blame]2172 .pixel_tile = 1,
2173 .channel_tile = 8,
2174 };
Marat Dukhan37c83512020-06-29 13:25:53 -0700[diff] [blame]2175 xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__wasmsimd_x8;
Marat Dukhanc303fe62020-06-26 10:09:25 -0700[diff] [blame]2176 if (is_wasm_x86) {
Marat Dukhan3fa52c82020-07-08 12:54:33 -0700[diff] [blame]2177 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__wasmsimd_x86_x8;
Marat Dukhanc303fe62020-06-26 10:09:25 -0700[diff] [blame]2178 } else {
Marat Dukhan3fa52c82020-07-08 12:54:33 -0700[diff] [blame]2179 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__wasmsimd_arm_x8;
Marat Dukhanc303fe62020-06-26 10:09:25 -0700[diff] [blame]2180 }
Marat Dukhan9df9dc62020-07-10 20:08:49 -0700[diff] [blame]2181 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__wasmsimd_x16;
Marat Dukhanf4935a22020-07-16 15:59:10 -0700[diff] [blame]2182 if (is_wasm_x86) {
2183 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__wasmsimd_minmax_x8;
2184 } else {
2185 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__wasmsimd_bitselect_x8;
2186 }
Marat Dukhan37c83512020-06-29 13:25:53 -0700[diff] [blame]2187 xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__wasmsimd_x8;
Frank Barchard37297a62020-08-27 12:15:10 -0700[diff] [blame]2188 xnn_params.f32.relu = (xnn_univector_ukernel_function) xnn_f32_relu_ukernel__wasmsimd_x16;
Marat Dukhanb82b2cd2020-07-16 02:23:42 -0700[diff] [blame]2189 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__wasmsimd_addsub_x8;
2190 if (is_wasm_x86) {
2191 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__wasmsimd_addsub_x8;
2192 } else {
2193 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__wasmsimd_cvt_x8;
2194 }
2195 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__wasmsimd_addsub_x8;
2196 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__wasmsimd_addsub_x8;
Marat Dukhanb3635ed2020-07-16 12:36:28 -0700[diff] [blame]2197 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__wasmsimd_p5_div_x16;
Marat Dukhan37c83512020-06-29 13:25:53 -0700[diff] [blame]2198 xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__wasmsimd_x8;
Marat Dukhan6804bbd2020-06-30 19:26:11 -0700[diff] [blame]2199 xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__wasmsimd_sqrt_x8;
Marat Dukhan195f8eb2020-06-25 12:50:57 -0700[diff] [blame]2200 if (is_wasm_x86) {
2201 xnn_params.f32.prelu = (struct prelu_parameters) {
Marat Dukhan78299282020-07-15 17:38:06 -0700[diff] [blame]2202 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__wasmsimd_minmax_2x8,
Marat Dukhan195f8eb2020-06-25 12:50:57 -0700[diff] [blame]2203 .row_tile = 2,
2204 .channel_tile = 8,
2205 };
2206 } else {
2207 xnn_params.f32.prelu = (struct prelu_parameters) {
Marat Dukhan78299282020-07-15 17:38:06 -0700[diff] [blame]2208 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__wasmsimd_bitselect_2x8,
Marat Dukhan195f8eb2020-06-25 12:50:57 -0700[diff] [blame]2209 .row_tile = 2,
2210 .channel_tile = 8,
2211 };
2212 }
Marat Dukhan52238f02020-07-16 15:30:28 -0700[diff] [blame]2213 xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__wasmsimd_p5_x16_acc2;
Marat Dukhan8c417962020-07-08 12:27:50 -0700[diff] [blame]2214 xnn_params.f32.rmax = xnn_f32_rmax_ukernel__wasmsimd_arm;
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2215 if (is_wasm_x86) {
2216 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2217 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__wasmsimd_x86_x16,
2218 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasmsimd_x86_x16,
2219 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasmsimd_x86_x16,
2220 .linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_ukernel__wasmsimd_x16,
2221 .linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_ukernel__wasmsimd_x16,
2222 .linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_ukernel__wasmsimd_x16,
2223 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2224 };
2225 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]2226 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__wasmsimd_x86_x4,
2227 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__wasmsimd_x86_x4,
2228 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__wasmsimd_x86_x4,
2229 .linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_ukernel__wasmsimd_x4,
2230 .linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_ukernel__wasmsimd_x4,
2231 .linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_ukernel__wasmsimd_x4,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2232 .element_tile = 4,
2233 };
2234 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2235 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__wasmsimd_x86_x16,
2236 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasmsimd_x86_x16,
2237 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasmsimd_x86_x16,
2238 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2239 };
2240 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2241 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__wasmsimd_x86_x16,
2242 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasmsimd_x86_x16,
2243 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasmsimd_x86_x16,
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]2244
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2245 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2246 };
2247 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2248 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__wasmsimd_x86_x16,
2249 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasmsimd_x86_x16,
2250 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasmsimd_x86_x16,
2251 .linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_ukernel__wasmsimd_x16,
2252 .linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_ukernel__wasmsimd_x16,
2253 .linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_ukernel__wasmsimd_x16,
2254 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2255 };
2256 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2257 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__wasmsimd_x86_x16,
2258 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__wasmsimd_x86_x16,
2259 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__wasmsimd_x86_x16,
2260 .linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_ukernel__wasmsimd_x16,
2261 .linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_ukernel__wasmsimd_x16,
2262 .linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_ukernel__wasmsimd_x16,
2263 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2264 };
2265 } else {
2266 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2267 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__wasmsimd_arm_x16,
2268 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasmsimd_arm_x16,
2269 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasmsimd_arm_x16,
2270 .linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_ukernel__wasmsimd_x16,
2271 .linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_ukernel__wasmsimd_x16,
2272 .linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_ukernel__wasmsimd_x16,
2273 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2274 };
2275 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]2276 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__wasmsimd_arm_x4,
2277 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__wasmsimd_arm_x4,
2278 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__wasmsimd_arm_x4,
2279 .linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_ukernel__wasmsimd_x4,
2280 .linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_ukernel__wasmsimd_x4,
2281 .linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_ukernel__wasmsimd_x4,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2282 .element_tile = 4,
2283 };
2284 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2285 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__wasmsimd_arm_x16,
2286 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasmsimd_arm_x16,
2287 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasmsimd_arm_x16,
2288 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2289 };
2290 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2291 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__wasmsimd_arm_x16,
2292 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasmsimd_arm_x16,
2293 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasmsimd_arm_x16,
2294 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2295 };
2296 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2297 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__wasmsimd_arm_x16,
2298 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasmsimd_arm_x16,
2299 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasmsimd_arm_x16,
2300 .linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_ukernel__wasmsimd_x16,
2301 .linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_ukernel__wasmsimd_x16,
2302 .linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_ukernel__wasmsimd_x16,
2303 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2304 };
2305 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2306 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__wasmsimd_arm_x16,
2307 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__wasmsimd_arm_x16,
2308 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__wasmsimd_arm_x16,
2309 .linear.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_ukernel__wasmsimd_x16,
2310 .linear.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_ukernel__wasmsimd_x16,
2311 .linear.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_ukernel__wasmsimd_x16,
2312 .element_tile = 16,
Marat Dukhancdc56552020-06-26 19:49:41 -0700[diff] [blame]2313 };
2314 }
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]2315 xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2316 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__wasmsimd_x16,
2317 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__wasmsimd_x16,
2318 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__wasmsimd_x16,
2319 .element_tile = 16,
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]2320 };
Marat Dukhand816f622020-07-15 10:14:39 -0700[diff] [blame]2321 if (is_wasm_x86) {
2322 xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan9531e9f2020-07-24 15:25:02 -0700[diff] [blame]2323 .ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__wasmsimd_x86_2x,
Marat Dukhand816f622020-07-15 10:14:39 -0700[diff] [blame]2324 .channel_tile = 4,
2325 .row_tile = 2,
2326 };
2327 } else {
2328 xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan9531e9f2020-07-24 15:25:02 -0700[diff] [blame]2329 .ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c4__wasmsimd_arm_2x,
Marat Dukhand816f622020-07-15 10:14:39 -0700[diff] [blame]2330 .channel_tile = 4,
2331 .row_tile = 2,
2332 };
2333 }
Erich Elsen6e80fdc2020-06-09 15:35:37 -0700[diff] [blame]2334 #ifndef XNN_NO_NCHW_OPERATORS
2335 xnn_params.f32.spmm = (struct spmm_parameters) {
Frank Barchard9e053402020-10-19 15:29:08 -0700[diff] [blame]2336 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_16x1__wasmsimd_x86,
Erich Elsen6e80fdc2020-06-09 15:35:37 -0700[diff] [blame]2337 .mr = 16,
2338 .nr = 1,
2339 };
Erich Elsen0a1970e2020-06-10 09:24:59 -0700[diff] [blame]2340 xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
2341 .ukernel_with_symm_padding =
2342 (xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__psimd_2x2,
2343 .output_channel_tile = 4,
2344 .output_height_tile = 2,
2345 .output_width_tile = 2,
2346 };
Erich Elsen28928892020-06-12 08:08:19 -0700[diff] [blame]2347 xnn_params.f32.dwconv_chw_3x3 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]2348 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3p1__psimd_1x4_acc3,
Erich Elsen28928892020-06-12 08:08:19 -0700[diff] [blame]2349 .input_width_tile = 4,
2350 .output_width_tile = 4,
2351 .output_height_tile = 1,
2352 };
2353 xnn_params.f32.dwconv_chw_3x3s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]2354 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3s2p1__psimd_1x4_acc3,
Erich Elsen28928892020-06-12 08:08:19 -0700[diff] [blame]2355 .input_width_tile = 4,
2356 .output_width_tile = 4,
2357 .output_height_tile = 1,
2358 };
2359 xnn_params.f32.dwconv_chw_5x5 = (struct dwconv_chw_parameters) {
Marat Dukhan1c6cad92020-10-21 15:48:21 -0700[diff] [blame]2360 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_5x5p2__psimd_3x4,
Erich Elsen28928892020-06-12 08:08:19 -0700[diff] [blame]2361 .input_width_tile = 4,
2362 .output_width_tile = 4,
2363 .output_height_tile = 3,
2364 };
Erich Elsen7465a892020-06-13 14:02:04 -0700[diff] [blame]2365 xnn_params.f32.dwconv_chw_5x5s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]2366 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_5x5s2p2__psimd_1x4_acc2,
Erich Elsen7465a892020-06-13 14:02:04 -0700[diff] [blame]2367 .input_width_tile = 4,
2368 .output_width_tile = 4,
2369 .output_height_tile = 1,
2370 };
Marat Dukhanc5045bf2020-07-27 18:16:35 -0700[diff] [blame]2371 if (is_wasm_x86) {
2372 xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
2373 .ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__wasmsimd_x86_x4,
2374 .channel_tile = 4,
2375 };
2376 } else {
2377 xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
2378 .ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__wasmsimd_arm_x4,
2379 .channel_tile = 4,
2380 };
2381 }
Erich Elsen6e80fdc2020-06-09 15:35:37 -0700[diff] [blame]2382 #endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2383 #endif // XNN_NO_F32_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2384
2385 /**************************** X32 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2386 #ifndef XNN_NO_X32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2387 init_flags |= XNN_INIT_FLAG_X32;
2388
Marat Dukhan4662b192020-05-21 15:52:03 -0700[diff] [blame]2389 xnn_params.x32.fill = (struct fill_parameters) {
Marat Dukhan8ee37012020-07-16 13:17:13 -0700[diff] [blame]2390 .ukernel = (xnn_fill_ukernel_function) xnn_x32_fill_ukernel__wasmsimd,
Marat Dukhan4662b192020-05-21 15:52:03 -0700[diff] [blame]2391 .row_tile = 1,
2392 };
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2393 xnn_params.x32.pad = (struct pad_parameters) {
Marat Dukhan9306ae02020-07-16 15:51:13 -0700[diff] [blame]2394 .ukernel = (xnn_pad_ukernel_function) xnn_x32_pad_ukernel__wasmsimd,
Marat Dukhan63523d42020-05-22 17:07:33 -0700[diff] [blame]2395 .row_tile = 1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2396 };
Marat Dukhan9d4bfa22020-07-16 19:07:04 -0700[diff] [blame]2397 xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__wasmsimd;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2398 xnn_params.x32.zip = (struct zip_parameters) {
Marat Dukhane3b78762020-07-16 20:02:58 -0700[diff] [blame]2399 .x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__wasmsimd,
2400 .x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__wasmsimd,
2401 .x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__wasmsimd,
2402 .xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__wasmsimd,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2403 };
2404 #endif // XNN_NO_X32_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2405
Marat Dukhana199d492020-07-24 15:01:25 -0700[diff] [blame]2406#elif XNN_ARCH_WASM
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]2407 /**************************** QU8 micro-kernels ****************************/
2408 #ifndef XNN_NO_QU8_OPERATORS
2409 init_flags |= XNN_INIT_FLAG_QU8;
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2410
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]2411 xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_ukernel_2x2__scalar);
2412 xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_ukernel_2x2__scalar);
2413 xnn_params.qu8.gemm.mr = 2;
2414 xnn_params.qu8.gemm.nr = 2;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2415
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]2416 xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_ukernel_up1x9__scalar;
2417 xnn_params.qu8.dwconv[0].channel_tile = 1;
2418 xnn_params.qu8.dwconv[0].primary_tile = 9;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2419
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]2420 xnn_params.qu8.avgpool = (struct avgpool_parameters) {
2421 .up = (xnn_avgpool_unipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9x__scalar_c1,
2422 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_qu8_avgpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2423 .mr = 9,
2424 .qr = 8,
2425 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]2426 xnn_params.qu8.gavgpool = (struct gavgpool_parameters) {
2427 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7x__scalar_c1,
2428 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_qu8_gavgpool_minmax_ukernel_7p7x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2429 .mr = 7,
2430 };
Marat Dukhan08b7a972020-07-14 18:17:29 -0700[diff] [blame]2431 xnn_params.qu8.vadd = (xnn_vadd_ukernel_function) xnn_qu8_vadd_minmax_ukernel__scalar;
2432 #endif // XNN_NO_QU8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2433
2434 /**************************** U8 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2435 #ifndef XNN_NO_U8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2436 init_flags |= XNN_INIT_FLAG_U8;
2437
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2438 xnn_params.u8.maxpool = (struct maxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2439 .ukernel = (xnn_maxpool_ukernel_function) xnn_u8_maxpool_minmax_ukernel_9p8x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2440 .mr = 9,
2441 .qr = 8,
2442 };
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]2443 xnn_params.u8.clamp = (xnn_univector_ukernel_function) xnn_u8_clamp_ukernel__scalar_x4;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2444 xnn_params.u8.lut32norm = xnn_u8_lut32norm_ukernel__scalar;
2445 xnn_params.u8.rmax = xnn_u8_rmax_ukernel__scalar;
2446 #endif // XNN_NO_U8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2447
2448 /**************************** X8 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2449 #ifndef XNN_NO_X8_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2450 init_flags |= XNN_INIT_FLAG_X8;
2451
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2452 xnn_params.x8.lut = xnn_x8_lut_ukernel__scalar;
2453 xnn_params.x8.zip = (struct zip_parameters) {
2454 .x2 = (xnn_zipc_ukernel_function) xnn_x8_zip_x2_ukernel__scalar,
2455 .x3 = (xnn_zipc_ukernel_function) xnn_x8_zip_x3_ukernel__scalar,
2456 .x4 = (xnn_zipc_ukernel_function) xnn_x8_zip_x4_ukernel__scalar,
2457 .xm = (xnn_zipv_ukernel_function) xnn_x8_zip_xm_ukernel__scalar,
2458 };
2459 #endif // XNN_NO_X8_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2460
2461 /**************************** F32 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2462 #ifndef XNN_NO_F32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2463 init_flags |= XNN_INIT_FLAG_F32;
2464
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2465 if (is_wasm_x86) {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2466 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_2x4__scalar);
2467 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_2x4__scalar);
2468 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x4__wasm);
2469 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x4__wasm);
Marat Dukhan467f6362020-05-22 23:21:55 -0700[diff] [blame]2470 xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_2x4__scalar);
2471 xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_2x4__scalar);
2472 xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x4__wasm);
2473 xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x4__wasm);
Marat Dukhan869c62d2020-04-09 17:17:55 -0700[diff] [blame]2474 xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_2x4__scalar);
2475 xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_2x4__scalar);
2476 xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x4__wasm);
2477 xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x4__wasm);
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2478 xnn_params.f32.gemm.mr = 2;
2479 xnn_params.f32.gemm.nr = 4;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2480 } else {
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2481 xnn_params.f32.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x4__wasm);
2482 xnn_params.f32.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x4__wasm);
2483 xnn_params.f32.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_1x4__wasm);
2484 xnn_params.f32.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_1x4__wasm);
Marat Dukhan467f6362020-05-22 23:21:55 -0700[diff] [blame]2485 xnn_params.f32.gemm.relu.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_4x4__wasm);
2486 xnn_params.f32.gemm.relu.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_4x4__wasm);
2487 xnn_params.f32.gemm.relu.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_relu_ukernel_1x4__wasm);
2488 xnn_params.f32.gemm.relu.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_relu_ukernel_1x4__wasm);
Marat Dukhan869c62d2020-04-09 17:17:55 -0700[diff] [blame]2489 xnn_params.f32.gemm.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x4__wasm);
2490 xnn_params.f32.gemm.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x4__wasm);
2491 xnn_params.f32.gemm.linear.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_1x4__wasm);
2492 xnn_params.f32.gemm.linear.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_1x4__wasm);
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2493 xnn_params.f32.gemm.mr = 4;
2494 xnn_params.f32.gemm.nr = 4;
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2495 }
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2496 xnn_params.f32.gemm2.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_minmax_ukernel_4x2__wasm);
2497 xnn_params.f32.gemm2.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_minmax_ukernel_4x2__wasm),
Marat Dukhan869c62d2020-04-09 17:17:55 -0700[diff] [blame]2498 xnn_params.f32.gemm2.linear.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_f32_gemm_ukernel_4x2__wasm);
2499 xnn_params.f32.gemm2.linear.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_f32_igemm_ukernel_4x2__wasm),
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2500 xnn_params.f32.gemm2.mr = 4;
2501 xnn_params.f32.gemm2.nr = 2;
2502
2503 xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x4__wasm_acc2;
Marat Dukhan869c62d2020-04-09 17:17:55 -0700[diff] [blame]2504 xnn_params.f32.dwconv[0].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x4__wasm_acc2;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2505 xnn_params.f32.dwconv[0].channel_tile = 1;
2506 xnn_params.f32.dwconv[0].primary_tile = 4;
2507
2508 xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x9__wasm_acc2;
Marat Dukhan869c62d2020-04-09 17:17:55 -0700[diff] [blame]2509 xnn_params.f32.dwconv[1].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x9__wasm_acc2;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2510 xnn_params.f32.dwconv[1].channel_tile = 1;
2511 xnn_params.f32.dwconv[1].primary_tile = 9;
2512
2513 xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up1x25__wasm_acc2;
Marat Dukhan869c62d2020-04-09 17:17:55 -0700[diff] [blame]2514 xnn_params.f32.dwconv[2].linear.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_ukernel_up1x25__wasm_acc2;
Marat Dukhanaefaef32020-04-09 07:09:34 -0700[diff] [blame]2515 xnn_params.f32.dwconv[2].channel_tile = 1;
2516 xnn_params.f32.dwconv[2].primary_tile = 25;
2517
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2518 xnn_params.f32.avgpool = (struct avgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2519 .up = (xnn_avgpool_unipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9x__wasm_c1,
2520 .mp = (xnn_avgpool_multipass_ukernel_function) xnn_f32_avgpool_minmax_ukernel_9p8x__wasm_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2521 .mr = 9,
2522 .qr = 8,
2523 };
2524 xnn_params.f32.pavgpool = (struct pavgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2525 .up = (xnn_pavgpool_unipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9x__wasm_c1,
2526 .mp = (xnn_pavgpool_multipass_ukernel_function) xnn_f32_pavgpool_minmax_ukernel_9p8x__wasm_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2527 .mr = 9,
2528 .qr = 8,
2529 };
2530 xnn_params.f32.gavgpool = (struct gavgpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2531 .up = (xnn_gavgpool_unipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7x__wasm_c1,
2532 .mp = (xnn_gavgpool_multipass_ukernel_function) xnn_f32_gavgpool_minmax_ukernel_7p7x__wasm_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2533 .mr = 7,
2534 };
2535 xnn_params.f32.maxpool = (struct maxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2536 .ukernel = (xnn_maxpool_ukernel_function) xnn_f32_maxpool_minmax_ukernel_9p8x__wasm_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2537 .mr = 9,
2538 .qr = 8,
2539 };
2540 xnn_params.f32.argmaxpool[0] = (struct argmaxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2541 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_4x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2542 .mr = 4,
2543 };
2544 xnn_params.f32.argmaxpool[1] = (struct argmaxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2545 .up = (xnn_argmaxpool_unipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2546 .mr = 9,
2547 };
2548 xnn_params.f32.argmaxpool[2] = (struct argmaxpool_parameters) {
Marat Dukhan99936602020-04-11 16:47:01 -0700[diff] [blame]2549 .mp = (xnn_argmaxpool_multipass_ukernel_function) xnn_f32_argmaxpool_ukernel_9p8x__scalar_c1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2550 .mr = 9,
2551 .qr = 8,
2552 };
Marat Dukhan660fd192020-03-10 04:55:30 -0700[diff] [blame]2553 xnn_params.f32.ibilinear = (struct ibilinear_parameters) {
2554 .ukernel = (xnn_ibilinear_ukernel_function) xnn_f32_ibilinear_ukernel__scalar_c2,
Marat Dukhan69722492019-11-11 19:55:50 -0800[diff] [blame]2555 .pixel_tile = 1,
2556 .channel_tile = 2,
2557 };
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]2558 xnn_params.f32.abs = (xnn_univector_ukernel_function) xnn_f32_vabs_ukernel__scalar_x4;
Marat Dukhan5c5fa962020-03-10 18:38:33 -0700[diff] [blame]2559 xnn_params.f32.clamp = (xnn_univector_ukernel_function) xnn_f32_clamp_ukernel__wasm_x4;
Marat Dukhanc303fe62020-06-26 10:09:25 -0700[diff] [blame]2560 if (is_wasm_x86) {
2561 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__scalar_x4;
2562 } else {
2563 xnn_params.f32.hswish = (xnn_univector_ukernel_function) xnn_f32_hswish_ukernel__wasm_x4;
2564 }
Marat Dukhan28813332020-06-10 18:05:38 -0700[diff] [blame]2565 xnn_params.f32.lrelu = (xnn_univector_ukernel_function) xnn_f32_vlrelu_ukernel__scalar_x4;
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]2566 xnn_params.f32.neg = (xnn_univector_ukernel_function) xnn_f32_vneg_ukernel__scalar_x4;
Frank Barchard62c5e232020-07-21 17:42:19 -0700[diff] [blame]2567 if (is_wasm_x86) {
Frank Barchard37297a62020-08-27 12:15:10 -0700[diff] [blame]2568 xnn_params.f32.relu = (xnn_univector_ukernel_function) xnn_f32_relu_ukernel__scalar_x8;
Frank Barchard62c5e232020-07-21 17:42:19 -0700[diff] [blame]2569 } else {
Frank Barchard37297a62020-08-27 12:15:10 -0700[diff] [blame]2570 xnn_params.f32.relu = (xnn_univector_ukernel_function) xnn_f32_relu_ukernel__wasm_x8;
Frank Barchard62c5e232020-07-21 17:42:19 -0700[diff] [blame]2571 }
Marat Dukhan64e52512020-06-09 13:41:16 -0700[diff] [blame]2572 xnn_params.f32.rndne = (xnn_univector_ukernel_function) xnn_f32_vrndne_ukernel__scalar_libm_x4;
2573 xnn_params.f32.rndz = (xnn_univector_ukernel_function) xnn_f32_vrndz_ukernel__scalar_libm_x4;
2574 xnn_params.f32.rndu = (xnn_univector_ukernel_function) xnn_f32_vrndu_ukernel__scalar_libm_x4;
2575 xnn_params.f32.rndd = (xnn_univector_ukernel_function) xnn_f32_vrndd_ukernel__scalar_libm_x4;
Marat Dukhan3a77ea72019-12-23 12:10:24 -0800[diff] [blame]2576 xnn_params.f32.sigmoid = (xnn_univector_ukernel_function) xnn_f32_sigmoid_ukernel__scalar_lut64_p2_div_x2;
Marat Dukhan5020b962020-06-08 13:30:10 -0700[diff] [blame]2577 xnn_params.f32.sqr = (xnn_univector_ukernel_function) xnn_f32_vsqr_ukernel__scalar_x4;
Marat Dukhan6804bbd2020-06-30 19:26:11 -0700[diff] [blame]2578 xnn_params.f32.sqrt = (xnn_univector_ukernel_function) xnn_f32_vsqrt_ukernel__scalar_sqrt_x1;
Marat Dukhan7c1f8082020-06-25 13:26:20 -0700[diff] [blame]2579 if (is_wasm_x86) {
2580 xnn_params.f32.prelu = (struct prelu_parameters) {
2581 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__scalar_2x4,
2582 .row_tile = 2,
2583 .channel_tile = 4,
2584 };
2585 } else {
2586 xnn_params.f32.prelu = (struct prelu_parameters) {
2587 .ukernel = (xnn_prelu_ukernel_function) xnn_f32_prelu_ukernel__wasm_2x4,
2588 .row_tile = 2,
2589 .channel_tile = 4,
2590 };
2591 }
Marat Dukhan1edc4542020-01-27 12:40:13 -0800[diff] [blame]2592 xnn_params.f32.raddstoreexpminusmax = xnn_f32_raddstoreexpminusmax_ukernel__scalar_p5_x4_acc2;
2593 xnn_params.f32.rmax = xnn_f32_rmax_ukernel__scalar;
Marat Dukhanb1a0fc32019-12-02 19:32:02 -0800[diff] [blame]2594 xnn_params.f32.vadd = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2595 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vadd_minmax_ukernel__wasm_x8,
2596 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasm_x8,
2597 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vaddc_minmax_ukernel__wasm_x8,
Marat Dukhanb1a0fc32019-12-02 19:32:02 -0800[diff] [blame]2598 .element_tile = 8,
2599 };
Marat Dukhan69180502019-12-06 15:00:31 -0800[diff] [blame]2600 xnn_params.f32.vdiv = (struct vbinary_parameters) {
Frank Barchardc67dd7f2020-07-06 11:23:57 -0700[diff] [blame]2601 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdiv_minmax_ukernel__wasm_x2,
2602 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vdivc_minmax_ukernel__wasm_x2,
2603 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrdivc_minmax_ukernel__wasm_x2,
Marat Dukhan69180502019-12-06 15:00:31 -0800[diff] [blame]2604 .element_tile = 2,
2605 };
Marat Dukhan79e7f842019-12-05 14:35:50 -0800[diff] [blame]2606 xnn_params.f32.vmax = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2607 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmax_ukernel__wasm_x8,
2608 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasm_x8,
2609 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmaxc_ukernel__wasm_x8,
Marat Dukhan79e7f842019-12-05 14:35:50 -0800[diff] [blame]2610 .element_tile = 8,
2611 };
2612 xnn_params.f32.vmin = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2613 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmin_ukernel__wasm_x8,
2614 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasm_x8,
2615 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vminc_ukernel__wasm_x8,
Marat Dukhan79e7f842019-12-05 14:35:50 -0800[diff] [blame]2616 .element_tile = 8,
2617 };
Marat Dukhan1e782c42019-11-21 17:02:40 -0800[diff] [blame]2618 xnn_params.f32.vmul = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2619 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmul_minmax_ukernel__wasm_x8,
2620 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasm_x8,
2621 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vmulc_minmax_ukernel__wasm_x8,
Marat Dukhanca2733c2019-11-15 23:21:17 -0800[diff] [blame]2622 .element_tile = 8,
2623 };
Marat Dukhan05f3f6d2019-12-03 15:13:53 -0800[diff] [blame]2624 xnn_params.f32.vsub = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2625 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsub_minmax_ukernel__wasm_x8,
2626 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsubc_minmax_ukernel__wasm_x8,
2627 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vrsubc_minmax_ukernel__wasm_x8,
Marat Dukhan05f3f6d2019-12-03 15:13:53 -0800[diff] [blame]2628 .element_tile = 8,
2629 };
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]2630 xnn_params.f32.vsqrdiff = (struct vbinary_parameters) {
Frank Barchard9c7308f2020-08-31 17:03:01 -0700[diff] [blame]2631 .minmax.op_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiff_ukernel__scalar_x8,
2632 .minmax.opc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
2633 .minmax.ropc_ukernel = (xnn_vbinary_ukernel_function) xnn_f32_vsqrdiffc_ukernel__scalar_x8,
Marat Dukhanf7399262020-06-05 10:58:44 -0700[diff] [blame]2634 .element_tile = 8,
2635 };
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2636 xnn_params.f32.vmulcaddc = (struct vmulcaddc_parameters) {
Marat Dukhan9531e9f2020-07-24 15:25:02 -0700[diff] [blame]2637 .ukernel = (xnn_vmulcaddc_ukernel_function) xnn_f32_vmulcaddc_minmax_ukernel_c1__wasm_2x,
Marat Dukhan49e6ee92019-11-06 15:55:29 -0800[diff] [blame]2638 .channel_tile = 1,
2639 .row_tile = 2,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2640 };
Marat Dukhanefc47b82019-11-18 09:25:38 -0800[diff] [blame]2641 #ifndef XNN_NO_NCHW_OPERATORS
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2642 xnn_params.f32.spmm = (struct spmm_parameters) {
Marat Dukhan355ab432020-04-09 19:01:52 -0700[diff] [blame]2643 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x1__scalar,
Marat Dukhanbff791e2019-10-24 11:05:37 -0700[diff] [blame]2644 .mr = 8,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2645 .nr = 1,
2646 };
Erich Elsenc6afd9b2019-10-24 16:10:53 -0700[diff] [blame]2647 xnn_params.f32.spmm2 = (struct spmm_parameters) {
Marat Dukhan355ab432020-04-09 19:01:52 -0700[diff] [blame]2648 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x2__scalar,
Erich Elsenc6afd9b2019-10-24 16:10:53 -0700[diff] [blame]2649 .mr = 8,
2650 .nr = 2,
2651 };
2652 xnn_params.f32.spmm4 = (struct spmm_parameters) {
Marat Dukhan355ab432020-04-09 19:01:52 -0700[diff] [blame]2653 .ukernel = (xnn_spmm_ukernel_function) xnn_f32_spmm_minmax_ukernel_8x4__scalar,
Erich Elsenc6afd9b2019-10-24 16:10:53 -0700[diff] [blame]2654 .mr = 8,
2655 .nr = 4,
2656 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]2657 xnn_params.f32.conv_hwc2chw_3x3c3s2 = (struct conv_hwc2chw_parameters) {
Marat Dukhan14fe0b22019-10-23 21:20:07 -0700[diff] [blame]2658 .ukernel_with_symm_padding =
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]2659 (xnn_conv_hwc2chw_ukernel_function) xnn_f32_conv_hwc2chw_ukernel_3x3s2p1c3x4__scalar_1x1,
Marat Dukhan14fe0b22019-10-23 21:20:07 -0700[diff] [blame]2660 .output_channel_tile = 4,
2661 .output_height_tile = 1,
2662 .output_width_tile = 1,
2663 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]2664 xnn_params.f32.dwconv_chw_3x3 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]2665 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3p1__scalar_1x1_acc3,
Marat Dukhan14fe0b22019-10-23 21:20:07 -0700[diff] [blame]2666 .input_width_tile = 1,
2667 .output_width_tile = 1,
2668 .output_height_tile = 1,
2669 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]2670 xnn_params.f32.dwconv_chw_3x3s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]2671 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_3x3s2p1__scalar_1x1_acc3,
Marat Dukhan14fe0b22019-10-23 21:20:07 -0700[diff] [blame]2672 .input_width_tile = 1,
2673 .output_width_tile = 1,
2674 .output_height_tile = 1,
2675 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]2676 xnn_params.f32.dwconv_chw_5x5 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]2677 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_5x5p2__scalar_1x1_acc5,
Marat Dukhana99918a2019-11-15 14:40:12 -0800[diff] [blame]2678 .input_width_tile = 1,
2679 .output_width_tile = 1,
2680 .output_height_tile = 1,
2681 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]2682 xnn_params.f32.dwconv_chw_5x5s2 = (struct dwconv_chw_parameters) {
Marat Dukhan6f469a52020-10-21 20:02:52 -0700[diff] [blame]2683 .ukernel = (xnn_dwconv_chw_ukernel_function) xnn_f32_dwconv_chw_ukernel_5x5s2p2__scalar_1x1_acc5,
Marat Dukhana99918a2019-11-15 14:40:12 -0800[diff] [blame]2684 .input_width_tile = 1,
2685 .output_width_tile = 1,
2686 .output_height_tile = 1,
2687 };
Marat Dukhan1f29b802020-05-15 23:46:39 -0700[diff] [blame]2688 xnn_params.f32.gavgpool_cw = (struct gavgpool_cw_parameters) {
2689 .ukernel = (xnn_gavgpool_cw_ukernel_function) xnn_f32_gavgpool_cw_ukernel__scalar_x1,
Marat Dukhan14fe0b22019-10-23 21:20:07 -0700[diff] [blame]2690 .channel_tile = 1,
2691 };
Marat Dukhanefc47b82019-11-18 09:25:38 -0800[diff] [blame]2692 #endif // XNN_NO_NCHW_OPERATORS
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2693 #endif // XNN_NO_F32_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2694
2695 /**************************** X32 micro-kernels ****************************/
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2696 #ifndef XNN_NO_X32_OPERATORS
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2697 init_flags |= XNN_INIT_FLAG_X32;
2698
Marat Dukhan4662b192020-05-21 15:52:03 -0700[diff] [blame]2699 xnn_params.x32.fill = (struct fill_parameters) {
2700 .ukernel = (xnn_fill_ukernel_function) xnn_x32_fill_ukernel__scalar_float,
2701 .row_tile = 1,
2702 };
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2703 xnn_params.x32.pad = (struct pad_parameters) {
Marat Dukhan63523d42020-05-22 17:07:33 -0700[diff] [blame]2704 .ukernel = (xnn_pad_ukernel_function) xnn_x32_pad_ukernel__scalar_float,
2705 .row_tile = 1,
Marat Dukhan8fe54e42019-10-10 14:12:59 -0700[diff] [blame]2706 };
2707 xnn_params.x32.unpool = (xnn_unpool_ukernel_function) xnn_x32_unpool_ukernel__scalar;
2708 xnn_params.x32.zip = (struct zip_parameters) {
2709 .x2 = (xnn_zipc_ukernel_function) xnn_x32_zip_x2_ukernel__scalar,
2710 .x3 = (xnn_zipc_ukernel_function) xnn_x32_zip_x3_ukernel__scalar,
2711 .x4 = (xnn_zipc_ukernel_function) xnn_x32_zip_x4_ukernel__scalar,
2712 .xm = (xnn_zipv_ukernel_function) xnn_x32_zip_xm_ukernel__scalar,
2713 };
2714 #endif // XNN_NO_X32_OPERATORS
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2715
2716#else
2717 #error "Unsupported architecture"
2718#endif
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2719 xnn_params.init_flags = init_flags;
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2720}
2721
Marat Dukhan57133c02020-04-13 00:54:59 -0700[diff] [blame]2722#ifdef _WIN32
2723 static BOOL CALLBACK init_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
2724 init();
2725 return TRUE;
2726 }
2727#endif
2728
Marat Dukhan04f03be2019-11-19 12:36:47 -0800[diff] [blame]2729enum xnn_status xnn_initialize(const struct xnn_allocator* allocator) {
Marat Dukhand343c222019-10-07 09:22:14 -0700[diff] [blame]2730 #ifndef __EMSCRIPTEN__
2731 if (!cpuinfo_initialize()) {
2732 return xnn_status_out_of_memory;
2733 }
2734 #endif
Marat Dukhan57133c02020-04-13 00:54:59 -0700[diff] [blame]2735 #ifdef _WIN32
2736 InitOnceExecuteOnce(&init_guard, &init_windows, NULL, NULL);
2737 #else
2738 pthread_once(&init_guard, &init);
2739 #endif
Marat Dukhan854fb6b2020-06-19 12:33:44 -0700[diff] [blame]2740 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) != 0) {
Marat Dukhan04f03be2019-11-19 12:36:47 -0800[diff] [blame]2741 if (allocator != NULL) {
2742 memcpy(&xnn_params.allocator, allocator, sizeof(struct xnn_allocator));
2743 } else {
2744 xnn_params.allocator.allocate = &xnn_allocate;
2745 xnn_params.allocator.reallocate = &xnn_reallocate;
2746 xnn_params.allocator.deallocate = &xnn_deallocate;
2747 xnn_params.allocator.aligned_allocate = &xnn_aligned_allocate;
2748 xnn_params.allocator.aligned_deallocate = &xnn_aligned_deallocate;
2749 }
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2750 return xnn_status_success;
2751 } else {
2752 return xnn_status_unsupported_hardware;
2753 }
2754}
2755
2756enum xnn_status xnn_deinitialize(void) {
Marat Dukhand343c222019-10-07 09:22:14 -0700[diff] [blame]2757 #ifndef __EMSCRIPTEN__
2758 cpuinfo_deinitialize();
2759 #endif
XNNPACK Teamb455b122019-09-27 18:10:33 -0700[diff] [blame]2760 return xnn_status_success;
2761}