WAsm SpMM microkernels unrolled by 2 and 4.
PiperOrigin-RevId: 338371416
diff --git a/src/f32-spmm/gen/16x1-minmax-wasmsimd-x86-unroll2.c b/src/f32-spmm/gen/16x1-minmax-wasmsimd-x86-unroll2.c
new file mode 100644
index 0000000..1dd45d7
--- /dev/null
+++ b/src/f32-spmm/gen/16x1-minmax-wasmsimd-x86-unroll2.c
@@ -0,0 +1,233 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-spmm/wasmsimd.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <wasm_simd128.h>
+
+#include <xnnpack/spmm.h>
+
+
+void xnn_f32_spmm_minmax_ukernel_16x1__wasmsimd_x86_unroll2(
+ uint32_t batch_size,
+ uint32_t output_channels,
+ const float*restrict input,
+ const float*restrict weights,
+ const int32_t*restrict widx_dmap,
+ const uint32_t*restrict nidx_nnzmap,
+ float*restrict output,
+ const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(batch_size != 0);
+
+ const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min);
+ const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max);
+ size_t n = batch_size;
+ while XNN_LIKELY(n >= 16) {
+ const float*restrict w = weights;
+ const int32_t* dmap = widx_dmap;
+ const uint32_t* nnzmap = nidx_nnzmap;
+ size_t c = output_channels;
+ do {
+ uint32_t nnz = *nnzmap++;
+ v128_t vacc0123x0 = wasm_v32x4_load_splat(w);
+ w += 1;
+ v128_t vacc0123x1 = wasm_f32x4_splat(0.0f);
+ v128_t vacc4567x0 = vacc0123x0;
+ v128_t vacc4567x1 = wasm_f32x4_splat(0.0f);
+ v128_t vacc89ABx0 = vacc0123x0;
+ v128_t vacc89ABx1 = wasm_f32x4_splat(0.0f);
+ v128_t vaccCDEFx0 = vacc0123x0;
+ v128_t vaccCDEFx1 = wasm_f32x4_splat(0.0f);
+ for (; nnz >= 2; nnz -= 2) {
+ const intptr_t diff0 = dmap[0];
+ const intptr_t diff1 = dmap[1];
+ dmap += 2;
+ const v128_t vi0123x0 = wasm_v128_load(input);
+ const v128_t vi4567x0 = wasm_v128_load(input + 4);
+ const v128_t vi89ABx0 = wasm_v128_load(input + 8);
+ const v128_t viCDEFx0 = wasm_v128_load(input + 12);
+ input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff0);
+ const v128_t vw0 = wasm_v32x4_load_splat(w);
+ w += 1;
+ vacc0123x0 = wasm_f32x4_add(vacc0123x0, wasm_f32x4_mul(vi0123x0, vw0));
+ vacc4567x0 = wasm_f32x4_add(vacc4567x0, wasm_f32x4_mul(vi4567x0, vw0));
+ vacc89ABx0 = wasm_f32x4_add(vacc89ABx0, wasm_f32x4_mul(vi89ABx0, vw0));
+ vaccCDEFx0 = wasm_f32x4_add(vaccCDEFx0, wasm_f32x4_mul(viCDEFx0, vw0));
+ const v128_t vi0123x1 = wasm_v128_load(input);
+ const v128_t vi4567x1 = wasm_v128_load(input + 4);
+ const v128_t vi89ABx1 = wasm_v128_load(input + 8);
+ const v128_t viCDEFx1 = wasm_v128_load(input + 12);
+ input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff1);
+ const v128_t vw1 = wasm_v32x4_load_splat(w);
+ w += 1;
+ vacc0123x1 = wasm_f32x4_add(vacc0123x1, wasm_f32x4_mul(vi0123x1, vw1));
+ vacc4567x1 = wasm_f32x4_add(vacc4567x1, wasm_f32x4_mul(vi4567x1, vw1));
+ vacc89ABx1 = wasm_f32x4_add(vacc89ABx1, wasm_f32x4_mul(vi89ABx1, vw1));
+ vaccCDEFx1 = wasm_f32x4_add(vaccCDEFx1, wasm_f32x4_mul(viCDEFx1, vw1));
+ }
+ v128_t vacc0123 = vacc0123x0;
+ v128_t vacc4567 = vacc4567x0;
+ v128_t vacc89AB = vacc89ABx0;
+ v128_t vaccCDEF = vaccCDEFx0;
+ vacc0123 = wasm_f32x4_add(vacc0123, vacc0123x1);
+ vacc4567 = wasm_f32x4_add(vacc4567, vacc4567x1);
+ vacc89AB = wasm_f32x4_add(vacc89AB, vacc89ABx1);
+ vaccCDEF = wasm_f32x4_add(vaccCDEF, vaccCDEFx1);
+ if XNN_LIKELY(nnz != 0) {
+ do {
+ const intptr_t diff = *dmap++;
+ const v128_t vi0123 = wasm_v128_load(input);
+ const v128_t vi4567 = wasm_v128_load(input + 4);
+ const v128_t vi89AB = wasm_v128_load(input + 8);
+ const v128_t viCDEF = wasm_v128_load(input + 12);
+ input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
+ const v128_t vw = wasm_v32x4_load_splat(w); w += 1;
+ vacc0123 = wasm_f32x4_add(vacc0123, wasm_f32x4_mul(vi0123, vw));
+ vacc4567 = wasm_f32x4_add(vacc4567, wasm_f32x4_mul(vi4567, vw));
+ vacc89AB = wasm_f32x4_add(vacc89AB, wasm_f32x4_mul(vi89AB, vw));
+ vaccCDEF = wasm_f32x4_add(vaccCDEF, wasm_f32x4_mul(viCDEF, vw));
+ } while (--nnz != 0);
+ }
+ v128_t vout0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax));
+ v128_t vout4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax));
+ v128_t vout89AB = wasm_v128_bitselect(vacc89AB, vmax, wasm_f32x4_le(vacc89AB, vmax));
+ v128_t voutCDEF = wasm_v128_bitselect(vaccCDEF, vmax, wasm_f32x4_le(vaccCDEF, vmax));
+ vout0123 = wasm_v128_bitselect(vmin, vout0123, wasm_f32x4_lt(vout0123, vmin));
+ vout4567 = wasm_v128_bitselect(vmin, vout4567, wasm_f32x4_lt(vout4567, vmin));
+ vout89AB = wasm_v128_bitselect(vmin, vout89AB, wasm_f32x4_lt(vout89AB, vmin));
+ voutCDEF = wasm_v128_bitselect(vmin, voutCDEF, wasm_f32x4_lt(voutCDEF, vmin));
+ wasm_v128_store(output, vout0123);
+ wasm_v128_store(output + 4, vout4567);
+ wasm_v128_store(output + 8, vout89AB);
+ wasm_v128_store(output + 12, voutCDEF);
+ output += 1 * batch_size;
+ } while (--c != 0);
+ output -= batch_size * output_channels;
+ output += 16;
+ input += 16;
+ n -= 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ if (n & 8) {
+ const float*restrict w = weights;
+ const int32_t* dmap = widx_dmap;
+ const uint32_t* nnzmap = nidx_nnzmap;
+ size_t c = output_channels;
+ do {
+ uint32_t nnz = *nnzmap++;
+ v128_t vacc0123 = wasm_v32x4_load_splat(w); w += 1;
+ v128_t vacc4567 = vacc0123;
+ if XNN_LIKELY(nnz != 0) {
+ do {
+ const intptr_t diff = *dmap++;
+ const v128_t vi0123 = wasm_v128_load(input);
+ const v128_t vi4567 = wasm_v128_load(input + 4);
+ input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
+ const v128_t vw = wasm_v32x4_load_splat(w); w += 1;
+ vacc0123 = wasm_f32x4_add(vacc0123, wasm_f32x4_mul(vi0123, vw));
+ vacc4567 = wasm_f32x4_add(vacc4567, wasm_f32x4_mul(vi4567, vw));
+ } while (--nnz != 0);
+ }
+ v128_t vout0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax));
+ v128_t vout4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax));
+ vout0123 = wasm_v128_bitselect(vmin, vout0123, wasm_f32x4_lt(vout0123, vmin));
+ vout4567 = wasm_v128_bitselect(vmin, vout4567, wasm_f32x4_lt(vout4567, vmin));
+ wasm_v128_store(output, vout0123);
+
+ wasm_v128_store(output + 4, vout4567);
+ output += 1 * batch_size;
+ } while (--c != 0);
+ output -= batch_size * output_channels;
+ output += 8;
+ input += 8;
+ }
+ if (n & 4) {
+ const float*restrict w = weights;
+ const int32_t* dmap = widx_dmap;
+ const uint32_t* nnzmap = nidx_nnzmap;
+ size_t c = output_channels;
+ do {
+ uint32_t nnz = *nnzmap++;
+ v128_t vacc0123 = wasm_v32x4_load_splat(w); w += 1;
+ if XNN_LIKELY(nnz != 0) {
+ do {
+ const intptr_t diff = *dmap++;
+ const v128_t vi0123 = wasm_v128_load(input);
+ input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
+ const v128_t vw = wasm_v32x4_load_splat(w); w += 1;
+ vacc0123 = wasm_f32x4_add(vacc0123, wasm_f32x4_mul(vi0123, vw));
+ } while (--nnz != 0);
+ }
+ v128_t vout0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax));
+ vout0123 = wasm_v128_bitselect(vmin, vout0123, wasm_f32x4_lt(vout0123, vmin));
+ wasm_v128_store(output, vout0123);
+
+ output += 1 * batch_size;
+ } while (--c != 0);
+ output -= batch_size * output_channels;
+ output += 4;
+ input += 4;
+ }
+ if (n & 2) {
+ const float*restrict w = weights;
+ const int32_t* dmap = widx_dmap;
+ const uint32_t* nnzmap = nidx_nnzmap;
+ size_t c = output_channels;
+ do {
+ uint32_t nnz = *nnzmap++;
+ v128_t vacc01 = wasm_v32x4_load_splat(w); w += 1;
+ if XNN_LIKELY(nnz != 0) {
+ do {
+ const intptr_t diff = *dmap++;
+ const v128_t vi01 = wasm_v64x2_load_splat(input);
+ input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
+ const v128_t vw = wasm_v32x4_load_splat(w); w += 1;
+ vacc01 = wasm_f32x4_add(vacc01, wasm_f32x4_mul(vi01, vw));
+ } while (--nnz != 0);
+ }
+ v128_t vout01 = wasm_v128_bitselect(vacc01, vmax, wasm_f32x4_le(vacc01, vmax));
+ vout01 = wasm_v128_bitselect(vmin, vout01, wasm_f32x4_lt(vout01, vmin));
+ *((double*) output) = wasm_f64x2_extract_lane(vout01, 0);
+
+ output += 1 * batch_size;
+ } while (--c != 0);
+ output -= batch_size * output_channels;
+ output += 2;
+ input += 2;
+ }
+ if (n & 1) {
+ const float*restrict w = weights;
+ const int32_t* dmap = widx_dmap;
+ const uint32_t* nnzmap = nidx_nnzmap;
+ size_t c = output_channels;
+ do {
+ uint32_t nnz = *nnzmap++;
+ v128_t vacc0 = wasm_v32x4_load_splat(w); w += 1;
+ if XNN_LIKELY(nnz != 0) {
+ do {
+ const intptr_t diff = *dmap++;
+ const v128_t vi0 = wasm_v32x4_load_splat(input);
+ input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
+ const v128_t vw = wasm_v32x4_load_splat(w); w += 1;
+ vacc0 = wasm_f32x4_add(vacc0, wasm_f32x4_mul(vi0, vw));
+ } while (--nnz != 0);
+ }
+ v128_t vout0 = wasm_v128_bitselect(vacc0, vmax, wasm_f32x4_le(vacc0, vmax));
+ vout0 = wasm_v128_bitselect(vmin, vout0, wasm_f32x4_lt(vout0, vmin));
+ *output = wasm_f32x4_extract_lane(vout0, 0);
+
+ output += 1 * batch_size;
+ } while (--c != 0);
+ output -= batch_size * output_channels;
+ output += 1;
+ input += 1;
+ }
+ }
+}