Auto-generate 5x5s2 DWCONV CHW micro-kernels
PiperOrigin-RevId: 339770846
diff --git a/BUILD.bazel b/BUILD.bazel
index ff2069d..934beb1 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -1241,6 +1241,16 @@
"src/f32-dwconv2d-chw/gen/5x5p2-minmax-neon-4x4-acc2.c",
"src/f32-dwconv2d-chw/gen/5x5p2-minmax-neon-4x4.c",
"src/f32-dwconv2d-chw/gen/5x5p2-minmax-neon-5x4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc2.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc3.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc5.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc2.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc3.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4-acc2.c",
"src/f32-gavgpool-cw/neon-x4.c",
"src/f32-gavgpool/7p7x-minmax-neon-c4.c",
"src/f32-gavgpool/7x-minmax-neon-c4.c",
@@ -1662,7 +1672,6 @@
"src/f32-conv-hwc/gen/3x3s2p1c3x8-neonfma-2x1.c",
"src/f32-conv-hwc/gen/3x3s2p1c3x8-neonfma-2x2.c",
"src/f32-conv-hwc2chw/3x3s2p1c3x4-neonfma-2x2.c",
- "src/f32-dwconv2d-chw/5x5s2p2-neonfma-1x4-acc2.c",
"src/f32-dwconv2d-chw/gen/3x3p1-minmax-neonfma-1x4-acc2.c",
"src/f32-dwconv2d-chw/gen/3x3p1-minmax-neonfma-1x4-acc3.c",
"src/f32-dwconv2d-chw/gen/3x3p1-minmax-neonfma-1x4-acc4.c",
@@ -1694,6 +1703,16 @@
"src/f32-dwconv2d-chw/gen/5x5p2-minmax-neonfma-4x4-acc2.c",
"src/f32-dwconv2d-chw/gen/5x5p2-minmax-neonfma-4x4.c",
"src/f32-dwconv2d-chw/gen/5x5p2-minmax-neonfma-5x4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc2.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc3.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc5.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc2.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc3.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4.c",
+ "src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4-acc2.c",
"src/f32-gemm/gen-inc/1x8inc-minmax-neonfma-lane-ld64.c",
"src/f32-gemm/gen-inc/4x8inc-minmax-neonfma-lane-ld64.c",
"src/f32-gemm/gen-inc/4x8inc-minmax-neonfma-lane-ld128.c",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2741116..9e3de4e 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -706,6 +706,16 @@
src/f32-dwconv2d-chw/gen/5x5p2-minmax-neon-4x4-acc2.c
src/f32-dwconv2d-chw/gen/5x5p2-minmax-neon-4x4.c
src/f32-dwconv2d-chw/gen/5x5p2-minmax-neon-5x4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc2.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc3.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc5.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc2.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc3.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4-acc2.c
src/f32-gavgpool-cw/neon-x4.c
src/f32-gavgpool/7p7x-minmax-neon-c4.c
src/f32-gavgpool/7x-minmax-neon-c4.c
@@ -1139,7 +1149,6 @@
src/f32-conv-hwc/gen/3x3s2p1c3x8-neonfma-2x1.c
src/f32-conv-hwc/gen/3x3s2p1c3x8-neonfma-2x2.c
src/f32-conv-hwc2chw/3x3s2p1c3x4-neonfma-2x2.c
- src/f32-dwconv2d-chw/5x5s2p2-neonfma-1x4-acc2.c
src/f32-dwconv2d-chw/gen/3x3p1-minmax-neonfma-1x4-acc2.c
src/f32-dwconv2d-chw/gen/3x3p1-minmax-neonfma-1x4-acc3.c
src/f32-dwconv2d-chw/gen/3x3p1-minmax-neonfma-1x4-acc4.c
@@ -1171,6 +1180,16 @@
src/f32-dwconv2d-chw/gen/5x5p2-minmax-neonfma-4x4-acc2.c
src/f32-dwconv2d-chw/gen/5x5p2-minmax-neonfma-4x4.c
src/f32-dwconv2d-chw/gen/5x5p2-minmax-neonfma-5x4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc2.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc3.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc5.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc2.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc3.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4.c
+ src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4-acc2.c
src/f32-gemm/gen-inc/1x8inc-minmax-neonfma-lane-ld64.c
src/f32-gemm/gen-inc/4x8inc-minmax-neonfma-lane-ld64.c
src/f32-gemm/gen-inc/4x8inc-minmax-neonfma-lane-ld128.c
diff --git a/bench/f32-dwconv2d-chw.cc b/bench/f32-dwconv2d-chw.cc
index cca7508..505e988 100644
--- a/bench/f32-dwconv2d-chw.cc
+++ b/bench/f32-dwconv2d-chw.cc
@@ -288,9 +288,36 @@
DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5p2__neonfma_4x4_acc2, 5, 5, 2, 1);
}
+ static void dwconv2d_chw_5x5s2p2__neonfma_1x4(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4, 5, 5, 2, 2);
+ }
+ static void dwconv2d_chw_5x5s2p2__neonfma_2x4(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4, 5, 5, 2, 2);
+ }
+ static void dwconv2d_chw_5x5s2p2__neonfma_3x4(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4, 5, 5, 2, 2);
+ }
static void dwconv2d_chw_5x5s2p2__neonfma_1x4_acc2(benchmark::State& state, const char* net) {
DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc2, 5, 5, 2, 2);
}
+ static void dwconv2d_chw_5x5s2p2__neonfma_1x4_acc3(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3, 5, 5, 2, 2);
+ }
+ static void dwconv2d_chw_5x5s2p2__neonfma_1x4_acc4(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4, 5, 5, 2, 2);
+ }
+ static void dwconv2d_chw_5x5s2p2__neonfma_1x4_acc5(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5, 5, 5, 2, 2);
+ }
+ static void dwconv2d_chw_5x5s2p2__neonfma_2x4_acc2(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2, 5, 5, 2, 2);
+ }
+ static void dwconv2d_chw_5x5s2p2__neonfma_2x4_acc3(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3, 5, 5, 2, 2);
+ }
+ static void dwconv2d_chw_5x5s2p2__neonfma_3x4_acc2(benchmark::State& state, const char* net) {
+ DWConv2DBenchmark(state, xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2, 5, 5, 2, 2);
+ }
BENCHMARK_DWCONV(dwconv2d_chw_3x3p1__neonfma_1x4)
BENCHMARK_DWCONV(dwconv2d_chw_3x3p1__neonfma_2x4)
@@ -326,7 +353,16 @@
BENCHMARK_DWCONV(dwconv2d_chw_5x5p2__neonfma_3x4_acc2)
BENCHMARK_DWCONV(dwconv2d_chw_5x5p2__neonfma_4x4_acc2)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_1x4)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_2x4)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_3x4)
BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_1x4_acc2)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_1x4_acc3)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_1x4_acc4)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_1x4_acc5)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_2x4_acc2)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_2x4_acc3)
+ BENCHMARK_DWCONV(dwconv2d_chw_5x5s2p2__neonfma_3x4_acc2)
#endif // XNN_ARCH_ARM64
#if XNN_ARCH_X86 || XNN_ARCH_X86_64
diff --git a/scripts/generate-f32-dwconv2d-chw.sh b/scripts/generate-f32-dwconv2d-chw.sh
index 96e9cdb..ba89366 100755
--- a/scripts/generate-f32-dwconv2d-chw.sh
+++ b/scripts/generate-f32-dwconv2d-chw.sh
@@ -79,6 +79,30 @@
tools/xngen src/f32-dwconv2d-chw/5x5p2-neon.c.in -D ROW_TILE=3 -D ACCUMULATORS=2 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5p2-minmax-neonfma-3x4-acc2.c
tools/xngen src/f32-dwconv2d-chw/5x5p2-neon.c.in -D ROW_TILE=4 -D ACCUMULATORS=2 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5p2-minmax-neonfma-4x4-acc2.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=1 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=2 -D ACCUMULATORS=1 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=3 -D ACCUMULATORS=1 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4.c
+
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=2 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc2.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=3 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc3.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=4 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc4.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=5 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc5.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=2 -D ACCUMULATORS=2 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc2.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=2 -D ACCUMULATORS=3 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc3.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=3 -D ACCUMULATORS=2 -D FMA=0 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4-acc2.c
+
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=1 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=2 -D ACCUMULATORS=1 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=3 -D ACCUMULATORS=1 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4.c
+
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=2 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc2.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=3 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc3.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=4 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc4.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=1 -D ACCUMULATORS=5 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc5.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=2 -D ACCUMULATORS=2 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc2.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=2 -D ACCUMULATORS=3 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc3.c
+tools/xngen src/f32-dwconv2d-chw/5x5s2p2-neon.c.in -D ROW_TILE=3 -D ACCUMULATORS=2 -D FMA=1 -o src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4-acc2.c
+
################################### x86 SSE ###################################
tools/xngen src/f32-dwconv2d-chw/3x3p1-sse.c.in -D ROW_TILE=1 -D ACCUMULATORS=1 -o src/f32-dwconv2d-chw/gen/3x3p1-minmax-sse-1x4.c
tools/xngen src/f32-dwconv2d-chw/3x3p1-sse.c.in -D ROW_TILE=2 -D ACCUMULATORS=1 -o src/f32-dwconv2d-chw/gen/3x3p1-minmax-sse-2x4.c
diff --git a/src/f32-dwconv2d-chw/5x5p2-neonfma-3x4.c b/src/f32-dwconv2d-chw/5x5p2-neonfma-3x4.c
deleted file mode 100644
index 8937029..0000000
--- a/src/f32-dwconv2d-chw/5x5p2-neonfma-3x4.c
+++ /dev/null
@@ -1,652 +0,0 @@
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <arm_neon.h>
-
-#include <xnnpack/dwconv.h>
-#include <xnnpack/math.h>
-
-
-void xnn_f32_dwconv2d_chw_ukernel_5x5p2__neonfma_3x4(
- size_t input_height,
- size_t input_width,
- const float* input,
- const float* weights,
- const float *zero,
- float* output,
- uint32_t padding_top,
- const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
-{
- assert(input_height != 0);
- assert(input_width != 0);
- assert(input_width % sizeof(float) == 0);
- assert(padding_top == 2);
-
- const uint32x4_t vmask = vld1q_u32(params->neon.mask);
- const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
- const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
-
- const float32x4_t vw0123 = vld1q_f32(weights);
- const float32x4_t vw4567 = vld1q_f32(weights + 4);
- const float32x4_t vw89AB = vld1q_f32(weights + 8);
- const float32x4_t vwCDEF = vld1q_f32(weights + 12);
- const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
- const float32x4_t vwKLMN = vld1q_f32(weights + 20);
- const float32x2_t vwOP = vld1_f32(weights + 24);
-
- const size_t input_decrement = round_up_po2(input_width, 4 * sizeof(float));
-
- const float* i0 = zero;
- const float* i1 = zero;
- const float* i2 = input;
- const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
- const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
- const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
- const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
-
- float* o0 = output;
- float* o1 = (float*) ((uintptr_t) o0 + input_width);
- float* o2 = (float*) ((uintptr_t) o1 + input_width);
-
- size_t output_height = input_height;
- do {
- if XNN_UNPREDICTABLE(output_height < 2) {
- i3 = zero;
- o1 = o0;
- }
- if XNN_UNPREDICTABLE(output_height <= 2) {
- i4 = zero;
- o2 = o1;
- }
- if XNN_UNPREDICTABLE(output_height < 4) {
- i5 = zero;
- }
- if XNN_UNPREDICTABLE(output_height <= 4) {
- i6 = zero;
- }
-
- float32x4_t vi0x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi1x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi2x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi3x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi4x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi5x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi6x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi0x4567 = vld1q_f32(i0); i0 += 4;
- float32x4_t vi1x4567 = vld1q_f32(i1); i1 += 4;
- float32x4_t vi2x4567 = vld1q_f32(i2); i2 += 4;
- float32x4_t vi3x4567 = vld1q_f32(i3); i3 += 4;
- float32x4_t vi4x4567 = vld1q_f32(i4); i4 += 4;
- float32x4_t vi5x4567 = vld1q_f32(i5); i5 += 4;
- float32x4_t vi6x4567 = vld1q_f32(i6); i6 += 4;
-
- size_t w = input_width;
- for (; w > 8 * sizeof(float); w -= 4 * sizeof(float)) {
- float32x4_t vo4567p0 = vdupq_laneq_f32(vw0123, 0);
- float32x4_t vo4567p1 = vdupq_laneq_f32(vw0123, 0);
- float32x4_t vo4567p2 = vdupq_laneq_f32(vw0123, 0);
-
- const float32x4_t vi0x89AB = vld1q_f32(i0); i0 += 4;
- const float32x4_t vi1x89AB = vld1q_f32(i1); i1 += 4;
- const float32x4_t vi2x89AB = vld1q_f32(i2); i2 += 4;
- const float32x4_t vi3x89AB = vld1q_f32(i3); i3 += 4;
- const float32x4_t vi4x89AB = vld1q_f32(i4); i4 += 4;
- const float32x4_t vi5x89AB = vld1q_f32(i5); i5 += 4;
- const float32x4_t vi6x89AB = vld1q_f32(i6); i6 += 4;
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x4567, vw0123, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x4567, vw0123, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x4567, vw0123, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x4567, vw89AB, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x4567, vw89AB, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x4567, vw89AB, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x4567, vwCDEF, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x4567, vwCDEF, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x4567, vwCDEF, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x4567, vwGHIJ, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x4567, vwGHIJ, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x4567, vwGHIJ, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x4567, vwKLMN, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x4567, vwKLMN, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x4567, vwKLMN, 3);
-
- const float32x4_t vi0x3456 = vextq_f32(vi0x0123, vi0x4567, 3);
- const float32x4_t vi1x3456 = vextq_f32(vi1x0123, vi1x4567, 3);
- const float32x4_t vi2x3456 = vextq_f32(vi2x0123, vi2x4567, 3);
- const float32x4_t vi3x3456 = vextq_f32(vi3x0123, vi3x4567, 3);
- const float32x4_t vi4x3456 = vextq_f32(vi4x0123, vi4x4567, 3);
- const float32x4_t vi5x3456 = vextq_f32(vi5x0123, vi5x4567, 3);
- const float32x4_t vi6x3456 = vextq_f32(vi6x0123, vi6x4567, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x3456, vw0123, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x3456, vw0123, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x3456, vw0123, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x3456, vw4567, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x3456, vw4567, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x3456, vw4567, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x3456, vwCDEF, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x3456, vwCDEF, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x3456, vwCDEF, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x3456, vwGHIJ, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x3456, vwGHIJ, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x3456, vwGHIJ, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x3456, vwKLMN, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x3456, vwKLMN, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x3456, vwKLMN, 2);
-
- const float32x4_t vi0x2345 = vextq_f32(vi0x0123, vi0x4567, 2);
- const float32x4_t vi1x2345 = vextq_f32(vi1x0123, vi1x4567, 2);
- const float32x4_t vi2x2345 = vextq_f32(vi2x0123, vi2x4567, 2);
- const float32x4_t vi3x2345 = vextq_f32(vi3x0123, vi3x4567, 2);
- const float32x4_t vi4x2345 = vextq_f32(vi4x0123, vi4x4567, 2);
- const float32x4_t vi5x2345 = vextq_f32(vi5x0123, vi5x4567, 2);
- const float32x4_t vi6x2345 = vextq_f32(vi6x0123, vi6x4567, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x2345, vw0123, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x2345, vw0123, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x2345, vw0123, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x2345, vw4567, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x2345, vw4567, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x2345, vw4567, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x2345, vw89AB, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x2345, vw89AB, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x2345, vw89AB, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x2345, vwGHIJ, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x2345, vwGHIJ, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x2345, vwGHIJ, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x2345, vwKLMN, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x2345, vwKLMN, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x2345, vwKLMN, 1);
-
- vi0x0123 = vi0x4567;
- vi1x0123 = vi1x4567;
- vi2x0123 = vi2x4567;
- vi3x0123 = vi3x4567;
- vi4x0123 = vi4x4567;
- vi5x0123 = vi5x4567;
- vi6x0123 = vi6x4567;
-
- const float32x4_t vi0x5678 = vextq_f32(vi0x4567, vi0x89AB, 1);
- const float32x4_t vi1x5678 = vextq_f32(vi1x4567, vi1x89AB, 1);
- const float32x4_t vi2x5678 = vextq_f32(vi2x4567, vi2x89AB, 1);
- const float32x4_t vi3x5678 = vextq_f32(vi3x4567, vi3x89AB, 1);
- const float32x4_t vi4x5678 = vextq_f32(vi4x4567, vi4x89AB, 1);
- const float32x4_t vi5x5678 = vextq_f32(vi5x4567, vi5x89AB, 1);
- const float32x4_t vi6x5678 = vextq_f32(vi6x4567, vi6x89AB, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x5678, vw4567, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x5678, vw4567, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x5678, vw4567, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x5678, vw89AB, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x5678, vw89AB, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x5678, vw89AB, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x5678, vwCDEF, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x5678, vwCDEF, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x5678, vwCDEF, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x5678, vwGHIJ, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x5678, vwGHIJ, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x5678, vwGHIJ, 3);
-
- vo4567p0 = vfmaq_lane_f32( vo4567p0, vi4x5678, vwOP, 0);
- vo4567p1 = vfmaq_lane_f32( vo4567p1, vi5x5678, vwOP, 0);
- vo4567p2 = vfmaq_lane_f32( vo4567p2, vi6x5678, vwOP, 0);
-
- const float32x4_t vi0x6789 = vextq_f32(vi0x4567, vi0x89AB, 2);
- const float32x4_t vi1x6789 = vextq_f32(vi1x4567, vi1x89AB, 2);
- const float32x4_t vi2x6789 = vextq_f32(vi2x4567, vi2x89AB, 2);
- const float32x4_t vi3x6789 = vextq_f32(vi3x4567, vi3x89AB, 2);
- const float32x4_t vi4x6789 = vextq_f32(vi4x4567, vi4x89AB, 2);
- const float32x4_t vi5x6789 = vextq_f32(vi5x4567, vi5x89AB, 2);
- const float32x4_t vi6x6789 = vextq_f32(vi6x4567, vi6x89AB, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x6789, vw4567, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x6789, vw4567, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x6789, vw4567, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x6789, vw89AB, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x6789, vw89AB, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x6789, vw89AB, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x6789, vwCDEF, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x6789, vwCDEF, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x6789, vwCDEF, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x6789, vwKLMN, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x6789, vwKLMN, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x6789, vwKLMN, 0);
-
- vo4567p0 = vfmaq_lane_f32( vo4567p0, vi4x6789, vwOP, 1);
- vo4567p1 = vfmaq_lane_f32( vo4567p1, vi5x6789, vwOP, 1);
- vo4567p2 = vfmaq_lane_f32( vo4567p2, vi6x6789, vwOP, 1);
-
- vi0x4567 = vi0x89AB;
- vi1x4567 = vi1x89AB;
- vi2x4567 = vi2x89AB;
- vi3x4567 = vi3x89AB;
- vi4x4567 = vi4x89AB;
- vi5x4567 = vi5x89AB;
- vi6x4567 = vi6x89AB;
-
- float32x4_t vo0 = vo4567p0;
- float32x4_t vo1 = vo4567p1;
- float32x4_t vo2 = vo4567p2;
-
- vo0 = vmaxq_f32(vo0, vmin);
- vo1 = vmaxq_f32(vo1, vmin);
- vo2 = vmaxq_f32(vo2, vmin);
-
- vo0 = vminq_f32(vo0, vmax);
- vo1 = vminq_f32(vo1, vmax);
- vo2 = vminq_f32(vo2, vmax);
-
- vst1q_f32(o2, vo2); o2 += 4;
- vst1q_f32(o1, vo1); o1 += 4;
- vst1q_f32(o0, vo0); o0 += 4;
- }
- // Always process the last block of 5..8 pixels.
- if XNN_LIKELY(w > 4 * sizeof(float))
- {
- float32x4_t vo4567p0 = vdupq_laneq_f32(vw0123, 0);
- float32x4_t vo4567p1 = vdupq_laneq_f32(vw0123, 0);
- float32x4_t vo4567p2 = vdupq_laneq_f32(vw0123, 0);
-
- float32x4_t vi0x89AB = vld1q_f32(i0); i0 += 4;
- float32x4_t vi1x89AB = vld1q_f32(i1); i1 += 4;
- float32x4_t vi2x89AB = vld1q_f32(i2); i2 += 4;
- float32x4_t vi3x89AB = vld1q_f32(i3); i3 += 4;
- float32x4_t vi4x89AB = vld1q_f32(i4); i4 += 4;
- float32x4_t vi5x89AB = vld1q_f32(i5); i5 += 4;
- float32x4_t vi6x89AB = vld1q_f32(i6); i6 += 4;
-
- vi0x89AB = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi0x89AB)));
- vi1x89AB = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi1x89AB)));
- vi2x89AB = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi2x89AB)));
- vi3x89AB = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi3x89AB)));
- vi4x89AB = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi4x89AB)));
- vi5x89AB = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi5x89AB)));
- vi6x89AB = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi6x89AB)));
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x4567, vw0123, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x4567, vw0123, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x4567, vw0123, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x4567, vw89AB, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x4567, vw89AB, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x4567, vw89AB, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x4567, vwCDEF, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x4567, vwCDEF, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x4567, vwCDEF, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x4567, vwGHIJ, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x4567, vwGHIJ, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x4567, vwGHIJ, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x4567, vwKLMN, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x4567, vwKLMN, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x4567, vwKLMN, 3);
-
- const float32x4_t vi0x3456 = vextq_f32(vi0x0123, vi0x4567, 3);
- const float32x4_t vi1x3456 = vextq_f32(vi1x0123, vi1x4567, 3);
- const float32x4_t vi2x3456 = vextq_f32(vi2x0123, vi2x4567, 3);
- const float32x4_t vi3x3456 = vextq_f32(vi3x0123, vi3x4567, 3);
- const float32x4_t vi4x3456 = vextq_f32(vi4x0123, vi4x4567, 3);
- const float32x4_t vi5x3456 = vextq_f32(vi5x0123, vi5x4567, 3);
- const float32x4_t vi6x3456 = vextq_f32(vi6x0123, vi6x4567, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x3456, vw0123, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x3456, vw0123, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x3456, vw0123, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x3456, vw4567, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x3456, vw4567, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x3456, vw4567, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x3456, vwCDEF, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x3456, vwCDEF, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x3456, vwCDEF, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x3456, vwGHIJ, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x3456, vwGHIJ, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x3456, vwGHIJ, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x3456, vwKLMN, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x3456, vwKLMN, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x3456, vwKLMN, 2);
-
- const float32x4_t vi0x2345 = vextq_f32(vi0x0123, vi0x4567, 2);
- const float32x4_t vi1x2345 = vextq_f32(vi1x0123, vi1x4567, 2);
- const float32x4_t vi2x2345 = vextq_f32(vi2x0123, vi2x4567, 2);
- const float32x4_t vi3x2345 = vextq_f32(vi3x0123, vi3x4567, 2);
- const float32x4_t vi4x2345 = vextq_f32(vi4x0123, vi4x4567, 2);
- const float32x4_t vi5x2345 = vextq_f32(vi5x0123, vi5x4567, 2);
- const float32x4_t vi6x2345 = vextq_f32(vi6x0123, vi6x4567, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x2345, vw0123, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x2345, vw0123, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x2345, vw0123, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x2345, vw4567, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x2345, vw4567, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x2345, vw4567, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x2345, vw89AB, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x2345, vw89AB, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x2345, vw89AB, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x2345, vwGHIJ, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x2345, vwGHIJ, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x2345, vwGHIJ, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x2345, vwKLMN, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x2345, vwKLMN, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x2345, vwKLMN, 1);
-
- vi0x0123 = vi0x4567;
- vi1x0123 = vi1x4567;
- vi2x0123 = vi2x4567;
- vi3x0123 = vi3x4567;
- vi4x0123 = vi4x4567;
- vi5x0123 = vi5x4567;
- vi6x0123 = vi6x4567;
-
- const float32x4_t vi0x5678 = vextq_f32(vi0x4567, vi0x89AB, 1);
- const float32x4_t vi1x5678 = vextq_f32(vi1x4567, vi1x89AB, 1);
- const float32x4_t vi2x5678 = vextq_f32(vi2x4567, vi2x89AB, 1);
- const float32x4_t vi3x5678 = vextq_f32(vi3x4567, vi3x89AB, 1);
- const float32x4_t vi4x5678 = vextq_f32(vi4x4567, vi4x89AB, 1);
- const float32x4_t vi5x5678 = vextq_f32(vi5x4567, vi5x89AB, 1);
- const float32x4_t vi6x5678 = vextq_f32(vi6x4567, vi6x89AB, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x5678, vw4567, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x5678, vw4567, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x5678, vw4567, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x5678, vw89AB, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x5678, vw89AB, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x5678, vw89AB, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x5678, vwCDEF, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x5678, vwCDEF, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x5678, vwCDEF, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x5678, vwGHIJ, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x5678, vwGHIJ, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x5678, vwGHIJ, 3);
-
- vo4567p0 = vfmaq_lane_f32( vo4567p0, vi4x5678, vwOP, 0);
- vo4567p1 = vfmaq_lane_f32( vo4567p1, vi5x5678, vwOP, 0);
- vo4567p2 = vfmaq_lane_f32( vo4567p2, vi6x5678, vwOP, 0);
-
- const float32x4_t vi0x6789 = vextq_f32(vi0x4567, vi0x89AB, 2);
- const float32x4_t vi1x6789 = vextq_f32(vi1x4567, vi1x89AB, 2);
- const float32x4_t vi2x6789 = vextq_f32(vi2x4567, vi2x89AB, 2);
- const float32x4_t vi3x6789 = vextq_f32(vi3x4567, vi3x89AB, 2);
- const float32x4_t vi4x6789 = vextq_f32(vi4x4567, vi4x89AB, 2);
- const float32x4_t vi5x6789 = vextq_f32(vi5x4567, vi5x89AB, 2);
- const float32x4_t vi6x6789 = vextq_f32(vi6x4567, vi6x89AB, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x6789, vw4567, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x6789, vw4567, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x6789, vw4567, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x6789, vw89AB, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x6789, vw89AB, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x6789, vw89AB, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x6789, vwCDEF, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x6789, vwCDEF, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x6789, vwCDEF, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x6789, vwKLMN, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x6789, vwKLMN, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x6789, vwKLMN, 0);
-
- vo4567p0 = vfmaq_lane_f32( vo4567p0, vi4x6789, vwOP, 1);
- vo4567p1 = vfmaq_lane_f32( vo4567p1, vi5x6789, vwOP, 1);
- vo4567p2 = vfmaq_lane_f32( vo4567p2, vi6x6789, vwOP, 1);
-
- vi0x4567 = vi0x89AB;
- vi1x4567 = vi1x89AB;
- vi2x4567 = vi2x89AB;
- vi3x4567 = vi3x89AB;
- vi4x4567 = vi4x89AB;
- vi5x4567 = vi5x89AB;
- vi6x4567 = vi6x89AB;
-
- float32x4_t vo0 = vo4567p0;
- float32x4_t vo1 = vo4567p1;
- float32x4_t vo2 = vo4567p2;
-
- vo0 = vmaxq_f32(vo0, vmin);
- vo0 = vminq_f32(vo0, vmax);
- vo1 = vmaxq_f32(vo1, vmin);
- vo1 = vminq_f32(vo1, vmax);
- vo2 = vmaxq_f32(vo2, vmin);
- vo2 = vminq_f32(vo2, vmax);
-
- vst1q_f32(o2, vo2); o2 += 4;
- vst1q_f32(o1, vo1); o1 += 4;
- vst1q_f32(o0, vo0); o0 += 4;
- w -= 4 * sizeof(float);
- }
- assert(w >= 1 * sizeof(float));
- assert(w <= 4 * sizeof(float));
- {
- float32x4_t vo4567p0 = vdupq_laneq_f32(vw0123, 0);
- float32x4_t vo4567p1 = vdupq_laneq_f32(vw0123, 0);
- float32x4_t vo4567p2 = vdupq_laneq_f32(vw0123, 0);
-
- // This might have already happened if there are more than 4 pixels, but
- // we can't count on it.
- vi0x4567 = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi0x4567)));
- vi1x4567 = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi1x4567)));
- vi2x4567 = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi2x4567)));
- vi3x4567 = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi3x4567)));
- vi4x4567 = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi4x4567)));
- vi5x4567 = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi5x4567)));
- vi6x4567 = vreinterpretq_f32_u32(vandq_u32(vmask, vreinterpretq_u32_f32(vi6x4567)));
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x4567, vw0123, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x4567, vw0123, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x4567, vw0123, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x4567, vw89AB, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x4567, vw89AB, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x4567, vw89AB, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x4567, vwCDEF, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x4567, vwCDEF, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x4567, vwCDEF, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x4567, vwGHIJ, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x4567, vwGHIJ, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x4567, vwGHIJ, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x4567, vwKLMN, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x4567, vwKLMN, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x4567, vwKLMN, 3);
-
- const float32x4_t vi0x3456 = vextq_f32(vi0x0123, vi0x4567, 3);
- const float32x4_t vi1x3456 = vextq_f32(vi1x0123, vi1x4567, 3);
- const float32x4_t vi2x3456 = vextq_f32(vi2x0123, vi2x4567, 3);
- const float32x4_t vi3x3456 = vextq_f32(vi3x0123, vi3x4567, 3);
- const float32x4_t vi4x3456 = vextq_f32(vi4x0123, vi4x4567, 3);
- const float32x4_t vi5x3456 = vextq_f32(vi5x0123, vi5x4567, 3);
- const float32x4_t vi6x3456 = vextq_f32(vi6x0123, vi6x4567, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x3456, vw0123, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x3456, vw0123, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x3456, vw0123, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x3456, vw4567, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x3456, vw4567, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x3456, vw4567, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x3456, vwCDEF, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x3456, vwCDEF, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x3456, vwCDEF, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x3456, vwGHIJ, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x3456, vwGHIJ, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x3456, vwGHIJ, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x3456, vwKLMN, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x3456, vwKLMN, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x3456, vwKLMN, 2);
-
- const float32x4_t vi0x2345 = vextq_f32(vi0x0123, vi0x4567, 2);
- const float32x4_t vi1x2345 = vextq_f32(vi1x0123, vi1x4567, 2);
- const float32x4_t vi2x2345 = vextq_f32(vi2x0123, vi2x4567, 2);
- const float32x4_t vi3x2345 = vextq_f32(vi3x0123, vi3x4567, 2);
- const float32x4_t vi4x2345 = vextq_f32(vi4x0123, vi4x4567, 2);
- const float32x4_t vi5x2345 = vextq_f32(vi5x0123, vi5x4567, 2);
- const float32x4_t vi6x2345 = vextq_f32(vi6x0123, vi6x4567, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x2345, vw0123, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x2345, vw0123, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x2345, vw0123, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x2345, vw4567, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x2345, vw4567, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x2345, vw4567, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x2345, vw89AB, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x2345, vw89AB, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x2345, vw89AB, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x2345, vwGHIJ, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x2345, vwGHIJ, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x2345, vwGHIJ, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi4x2345, vwKLMN, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi5x2345, vwKLMN, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi6x2345, vwKLMN, 1);
-
- const float32x4_t vzero = vmovq_n_f32(0.0f);
- const float32x4_t vi0x5678 = vextq_f32(vi0x4567, vzero, 1);
- const float32x4_t vi1x5678 = vextq_f32(vi1x4567, vzero, 1);
- const float32x4_t vi2x5678 = vextq_f32(vi2x4567, vzero, 1);
- const float32x4_t vi3x5678 = vextq_f32(vi3x4567, vzero, 1);
- const float32x4_t vi4x5678 = vextq_f32(vi4x4567, vzero, 1);
- const float32x4_t vi5x5678 = vextq_f32(vi5x4567, vzero, 1);
- const float32x4_t vi6x5678 = vextq_f32(vi6x4567, vzero, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x5678, vw4567, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x5678, vw4567, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x5678, vw4567, 0);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x5678, vw89AB, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x5678, vw89AB, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x5678, vw89AB, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x5678, vwCDEF, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x5678, vwCDEF, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x5678, vwCDEF, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x5678, vwGHIJ, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x5678, vwGHIJ, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x5678, vwGHIJ, 3);
-
- vo4567p0 = vfmaq_lane_f32( vo4567p0, vi4x5678, vwOP, 0);
- vo4567p1 = vfmaq_lane_f32( vo4567p1, vi5x5678, vwOP, 0);
- vo4567p2 = vfmaq_lane_f32( vo4567p2, vi6x5678, vwOP, 0);
-
- const float32x4_t vi0x6789 = vextq_f32(vi0x4567, vzero, 2);
- const float32x4_t vi1x6789 = vextq_f32(vi1x4567, vzero, 2);
- const float32x4_t vi2x6789 = vextq_f32(vi2x4567, vzero, 2);
- const float32x4_t vi3x6789 = vextq_f32(vi3x4567, vzero, 2);
- const float32x4_t vi4x6789 = vextq_f32(vi4x4567, vzero, 2);
- const float32x4_t vi5x6789 = vextq_f32(vi5x4567, vzero, 2);
- const float32x4_t vi6x6789 = vextq_f32(vi6x4567, vzero, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi0x6789, vw4567, 1);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi1x6789, vw4567, 1);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi2x6789, vw4567, 1);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi1x6789, vw89AB, 2);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi2x6789, vw89AB, 2);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi3x6789, vw89AB, 2);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi2x6789, vwCDEF, 3);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi3x6789, vwCDEF, 3);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi4x6789, vwCDEF, 3);
-
- vo4567p0 = vfmaq_laneq_f32(vo4567p0, vi3x6789, vwKLMN, 0);
- vo4567p1 = vfmaq_laneq_f32(vo4567p1, vi4x6789, vwKLMN, 0);
- vo4567p2 = vfmaq_laneq_f32(vo4567p2, vi5x6789, vwKLMN, 0);
-
- vo4567p0 = vfmaq_lane_f32(vo4567p0, vi4x6789, vwOP, 1);
- vo4567p1 = vfmaq_lane_f32(vo4567p1, vi5x6789, vwOP, 1);
- vo4567p2 = vfmaq_lane_f32(vo4567p2, vi6x6789, vwOP, 1);
-
- float32x4_t vo0 = vo4567p0;
- float32x4_t vo1 = vo4567p1;
- float32x4_t vo2 = vo4567p2;
-
- vo0 = vmaxq_f32(vo0, vmin);
- vo1 = vmaxq_f32(vo1, vmin);
- vo2 = vmaxq_f32(vo2, vmin);
-
- vo0 = vminq_f32(vo0, vmax);
- vo1 = vminq_f32(vo1, vmax);
- vo2 = vminq_f32(vo2, vmax);
-
- if XNN_LIKELY(w & (4 * sizeof(float))) {
- vst1q_f32(o2, vo2); o2 += 4;
- vst1q_f32(o1, vo1); o1 += 4;
- vst1q_f32(o0, vo0); o0 += 4;
- } else {
- float32x2_t vo0_lo = vget_low_f32(vo0);
- float32x2_t vo1_lo = vget_low_f32(vo1);
- float32x2_t vo2_lo = vget_low_f32(vo2);
- if (w & (2 * sizeof(float))) {
- vst1_f32(o2, vo2_lo); o2 += 2;
- vst1_f32(o1, vo1_lo); o1 += 2;
- vst1_f32(o0, vo0_lo); o0 += 2;
- vo0_lo = vget_high_f32(vo0);
- vo1_lo = vget_high_f32(vo1);
- vo2_lo = vget_high_f32(vo2);
- }
- if (w & (1 * sizeof(float))) {
- vst1_lane_f32(o2, vo2_lo, 0); o2 += 1;
- vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
- vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
- }
- }
- }
-
- i0 = (const float*) ((uintptr_t) i3 - input_decrement);
- i1 = (const float*) ((uintptr_t) i4 - input_decrement);
- i2 = (const float*) ((uintptr_t) i1 + input_width);
- i3 = (const float*) ((uintptr_t) i2 + input_width);
- i4 = (const float*) ((uintptr_t) i3 + input_width);
- i5 = (const float*) ((uintptr_t) i4 + input_width);
- i6 = (const float*) ((uintptr_t) i5 + input_width);
-
- o0 = o2;
- o1 = (float*) ((uintptr_t) o0 + input_width);
- o2 = (float*) ((uintptr_t) o1 + input_width);
-
- output_height = doz(output_height, 3);
- } while (output_height != 0);
-}
diff --git a/src/f32-dwconv2d-chw/5x5s2p2-neon.c.in b/src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
new file mode 100644
index 0000000..f5e74a9
--- /dev/null
+++ b/src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
@@ -0,0 +1,379 @@
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+$assert ROW_TILE >= 1
+$assert ACCUMULATORS >= 1
+$VMULADDQ_LANE_F32 = "vfmaq_lane_f32" if FMA else "vmlaq_lane_f32"
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__${"neonfma" if FMA else "neon"}_${ROW_TILE}x4${"_acc%d" % ACCUMULATORS if ACCUMULATORS > 1 else ""}(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ $for M in range(3, 3 + 2 * ROW_TILE):
+ const float* i${M} = (const float*) ((uintptr_t) i${M-1} + input_width);
+
+ $if ROW_TILE > 1:
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ $for M in range(1, ROW_TILE):
+ float* o${M} = (float*) ((uintptr_t) o${M-1} + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ $for M in range(3, 3 + 2 * ROW_TILE):
+ if XNN_UNPREDICTABLE(padded_input_height < ${3 + M}) {
+ i${M} = zero;
+ $if M % 2 == 0 and M <= 2 * ROW_TILE + 1:
+ o${M / 2 - 1} = o${M / 2 - 2};
+ }
+
+ $for M in range(3 + 2 * ROW_TILE):
+ float32x4_t vi${M}x0246 = vmovq_n_f32(0.0f);
+
+ $for M in range(3 + 2 * ROW_TILE):
+ float32x4_t vi${M}x1357 = vmovq_n_f32(0.0f);
+
+ $for M in range(3 + 2 * ROW_TILE):
+ float32x4x2_t vi${M}x8ACE9BDF = vld2q_f32(i${M}); i${M} += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ $for M in range(ROW_TILE):
+ float32x4_t vo${M}p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 1:
+ float32x4_t vo${M}p1 = vmulq_lane_f32(vi${2*M}x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ $else:
+ vo${M}p0 = ${VMULADDQ_LANE_F32}(vo${M}p0, vi${2*M}x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 2:
+ float32x4_t vo${M}p2 = vmulq_lane_f32(vi${2*M+1}x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ $else:
+ vo${M}p0 = ${VMULADDQ_LANE_F32}(vo${M}p0, vi${2*M+1}x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 3:
+ float32x4_t vo${M}p3 = vmulq_lane_f32(vi${2*M+2}x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ $else:
+ vo${M}p${4 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${4 % ACCUMULATORS}, vi${2*M+2}x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 4:
+ float32x4_t vo${M}p4 = vmulq_lane_f32(vi${2*M+3}x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ $else:
+ vo${M}p${5 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${5 % ACCUMULATORS}, vi${2*M+3}x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 5:
+ vo${M}p5 = vmulq_lane_f32(vi${2*M+4}x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ $else:
+ vo${M}p${6 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${6 % ACCUMULATORS}, vi${2*M+4}x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${7 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${7 % ACCUMULATORS}, vi${2*M}x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${8 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${8 % ACCUMULATORS}, vi${2*M+1}x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${9 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${9 % ACCUMULATORS}, vi${2*M+2}x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${10 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${10 % ACCUMULATORS}, vi${2*M+3}x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${11 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${11 % ACCUMULATORS}, vi${2*M+4}x8ACE9BDF.val[1], vwOP, 0);
+
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4_t vi${M}x68AC = vextq_f32(vi${M}x0246, vi${M}x8ACE9BDF.val[0], 3);
+ vi${M}x0246 = vi${M}x8ACE9BDF.val[0];
+
+ $for M in range(ROW_TILE):
+ vo${M}p${12 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${12 % ACCUMULATORS}, vi${2*M}x68AC, vget_low_f32(vw0123), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${13 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${13 % ACCUMULATORS}, vi${2*M+1}x68AC, vget_high_f32(vw4567), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${14 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${14 % ACCUMULATORS}, vi${2*M+2}x68AC, vget_high_f32(vw89AB), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${15 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${15 % ACCUMULATORS}, vi${2*M+3}x68AC, vget_low_f32(vwGHIJ), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${16 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${16 % ACCUMULATORS}, vi${2*M+4}x68AC, vget_low_f32(vwKLMN), 1);
+
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4_t vi${M}x79BD = vextq_f32(vi${M}x1357, vi${M}x8ACE9BDF.val[1], 3);
+ vi${M}x1357 = vi${M}x8ACE9BDF.val[1];
+
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4x2_t vi${M}xGIKMHJLN = vld2q_f32(i${M}); i${M} += 8;
+
+ $for M in range(ROW_TILE):
+ vo${M}p${17 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${17 % ACCUMULATORS}, vi${2*M}x79BD, vget_high_f32(vw0123), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${18 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${18 % ACCUMULATORS}, vi${2*M+1}x79BD, vget_high_f32(vw4567), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${19 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${19 % ACCUMULATORS}, vi${2*M+2}x79BD, vget_low_f32(vwCDEF), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${20 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${20 % ACCUMULATORS}, vi${2*M+3}x79BD, vget_low_f32(vwGHIJ), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${21 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${21 % ACCUMULATORS}, vi${2*M+4}x79BD, vget_high_f32(vwKLMN), 0);
+
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4_t vi${M}xACEG = vextq_f32(vi${M}x8ACE9BDF.val[0], vi${M}xGIKMHJLN.val[0], 1);
+ vi${M}x8ACE9BDF = vi${M}xGIKMHJLN;
+
+ $for M in range(ROW_TILE):
+ vo${M}p${22 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${22 % ACCUMULATORS}, vi${2*M}xACEG, vget_low_f32(vw4567), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${23 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${23 % ACCUMULATORS}, vi${2*M+1}xACEG, vget_high_f32(vw89AB), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${24 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${24 % ACCUMULATORS}, vi${2*M+2}xACEG, vget_high_f32(vwCDEF), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${25 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${25 % ACCUMULATORS}, vi${2*M+3}xACEG, vget_low_f32(vwKLMN), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${26 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${26 % ACCUMULATORS}, vi${2*M+4}xACEG, vwOP, 1);
+
+ $if ACCUMULATORS > 1:
+ $ACC_SLICE = 1
+ $while ACC_SLICE < ACCUMULATORS:
+ $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
+ $if A + ACC_SLICE < ACCUMULATORS:
+ $for M in range(ROW_TILE):
+ vo${M}p${A} = vaddq_f32(vo${M}p${A}, vo${M}p${A + ACC_SLICE});
+ $ACC_SLICE *= 2
+
+ $for M in range(ROW_TILE):
+ float32x4_t vo${M} = vmaxq_f32(vo${M}p0, vmin);
+
+ $for M in range(ROW_TILE):
+ vo${M} = vminq_f32(vo${M}, vmax);
+
+ $for M in reversed(range(ROW_TILE)):
+ vst1q_f32(o${M}, vo${M}); o${M} += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ $for M in range(ROW_TILE):
+ float32x4_t vo${M}p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4_t vi${M}x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi${M}x8ACE9BDF.val[0])));
+
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4_t vi${M}x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi${M}x8ACE9BDF.val[1])));
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 1:
+ float32x4_t vo${M}p1 = vmulq_lane_f32(vi${2*M}x8ACE, vget_high_f32(vw0123), 1);
+ $else:
+ vo${M}p0 = ${VMULADDQ_LANE_F32}(vo${M}p0, vi${2*M}x8ACE, vget_high_f32(vw0123), 1);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 2:
+ float32x4_t vo${M}p2 = vmulq_lane_f32(vi${2*M+1}x8ACE, vget_low_f32(vw89AB), 0);
+ $else:
+ vo${M}p0 = ${VMULADDQ_LANE_F32}(vo${M}p0, vi${2*M+1}x8ACE, vget_low_f32(vw89AB), 0);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 3:
+ float32x4_t vo${M}p3 = vmulq_lane_f32(vi${2*M+2}x8ACE, vget_low_f32(vwCDEF), 1);
+ $else:
+ vo${M}p${4 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${4 % ACCUMULATORS}, vi${2*M+2}x8ACE, vget_low_f32(vwCDEF), 1);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 4:
+ float32x4_t vo${M}p4 = vmulq_lane_f32(vi${2*M+3}x8ACE, vget_high_f32(vwGHIJ), 0);
+ $else:
+ vo${M}p${5 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${5 % ACCUMULATORS}, vi${2*M+3}x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ $for M in range(ROW_TILE):
+ $if ACCUMULATORS > 5:
+ vo${M}p5 = vmulq_lane_f32(vi${2*M+4}x8ACE, vget_high_f32(vwKLMN), 1);
+ $else:
+ vo${M}p${6 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${6 % ACCUMULATORS}, vi${2*M+4}x8ACE, vget_high_f32(vwKLMN), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${7 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${7 % ACCUMULATORS}, vi${2*M}x9BDF, vget_low_f32(vw4567), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${8 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${8 % ACCUMULATORS}, vi${2*M+1}x9BDF, vget_low_f32(vw89AB), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${9 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${9 % ACCUMULATORS}, vi${2*M+2}x9BDF, vget_high_f32(vwCDEF), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${10 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${10 % ACCUMULATORS}, vi${2*M+3}x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${11 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${11 % ACCUMULATORS}, vi${2*M+4}x9BDF, vwOP, 0);
+
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4_t vi${M}x68AC = vextq_f32(vi${M}x0246, vi${M}x8ACE, 3);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${12 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${12 % ACCUMULATORS}, vi${2*M}x68AC, vget_low_f32(vw0123), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${13 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${13 % ACCUMULATORS}, vi${2*M+1}x68AC, vget_high_f32(vw4567), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${14 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${14 % ACCUMULATORS}, vi${2*M+2}x68AC, vget_high_f32(vw89AB), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${15 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${15 % ACCUMULATORS}, vi${2*M+3}x68AC, vget_low_f32(vwGHIJ), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${16 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${16 % ACCUMULATORS}, vi${2*M+4}x68AC, vget_low_f32(vwKLMN), 1);
+
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4_t vi${M}x79BD = vextq_f32(vi${M}x1357, vi${M}x9BDF, 3);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${17 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${17 % ACCUMULATORS}, vi${2*M}x79BD, vget_high_f32(vw0123), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${18 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${18 % ACCUMULATORS}, vi${2*M+1}x79BD, vget_high_f32(vw4567), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${19 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${19 % ACCUMULATORS}, vi${2*M+2}x79BD, vget_low_f32(vwCDEF), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${20 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${20 % ACCUMULATORS}, vi${2*M+3}x79BD, vget_low_f32(vwGHIJ), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${21 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${21 % ACCUMULATORS}, vi${2*M+4}x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ $for M in range(3 + 2 * ROW_TILE):
+ const float32x4_t vi${M}xACEG = vextq_f32(vi${M}x8ACE, vzero, 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${22 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${22 % ACCUMULATORS}, vi${2*M}xACEG, vget_low_f32(vw4567), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${23 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${23 % ACCUMULATORS}, vi${2*M+1}xACEG, vget_high_f32(vw89AB), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${24 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${24 % ACCUMULATORS}, vi${2*M+2}xACEG, vget_high_f32(vwCDEF), 1);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${25 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${25 % ACCUMULATORS}, vi${2*M+3}xACEG, vget_low_f32(vwKLMN), 0);
+
+ $for M in range(ROW_TILE):
+ vo${M}p${26 % ACCUMULATORS} = ${VMULADDQ_LANE_F32}(vo${M}p${26 % ACCUMULATORS}, vi${2*M+4}xACEG, vwOP, 1);
+
+ $if ACCUMULATORS > 1:
+ $ACC_SLICE = 1
+ $while ACC_SLICE < ACCUMULATORS:
+ $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
+ $if A + ACC_SLICE < ACCUMULATORS:
+ $for M in range(ROW_TILE):
+ vo${M}p${A} = vaddq_f32(vo${M}p${A}, vo${M}p${A + ACC_SLICE});
+ $ACC_SLICE *= 2
+
+ $for M in range(ROW_TILE):
+ float32x4_t vo${M} = vmaxq_f32(vo${M}p0, vmin);
+
+ $for M in range(ROW_TILE):
+ vo${M} = vminq_f32(vo${M}, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ $for M in reversed(range(ROW_TILE)):
+ vst1q_f32(o${M}, vo${M}); o${M} += 4;
+ } else {
+ $for M in range(ROW_TILE):
+ float32x2_t vo${M}_lo = vget_low_f32(vo${M});
+ if (w_tmp & 2) {
+ $for M in reversed(range(ROW_TILE)):
+ vst1_f32(o${M}, vo${M}_lo); o${M} += 2;
+
+ $for M in range(ROW_TILE):
+ vo${M}_lo = vget_high_f32(vo${M});
+ }
+ if (w_tmp & 1) {
+ $for M in reversed(range(ROW_TILE)):
+ vst1_lane_f32(o${M}, vo${M}_lo, 0); o${M} += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i${2 * ROW_TILE} - input_decrement);
+ i1 = (const float*) ((uintptr_t) i${2 * ROW_TILE + 1} - input_decrement);
+ i2 = (const float*) ((uintptr_t) i${2 * ROW_TILE + 2} - input_decrement);
+ $for M in range(3, 3 + 2 * ROW_TILE):
+ i${M} = (const float*) ((uintptr_t) i${M-1} + input_width);
+
+ $if ROW_TILE > 1:
+ o0 = o${ROW_TILE - 1};
+ $for M in range(1, ROW_TILE):
+ o${M} = (float*) ((uintptr_t) o${M-1} + output_width);
+
+ $if ROW_TILE > 1:
+ output_height = doz(output_height, ${ROW_TILE});
+ padded_input_height = doz(padded_input_height, ${ROW_TILE * 2});
+ $else:
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/5x5s2p2-neonfma-1x4-acc2.c b/src/f32-dwconv2d-chw/5x5s2p2-neonfma-1x4-acc2.c
deleted file mode 100644
index 59c2e7b..0000000
--- a/src/f32-dwconv2d-chw/5x5s2p2-neonfma-1x4-acc2.c
+++ /dev/null
@@ -1,384 +0,0 @@
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <arm_neon.h>
-
-#include <xnnpack/dwconv.h>
-#include <xnnpack/math.h>
-
-
-void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc2(
- size_t input_height,
- size_t input_width,
- const float* input,
- const float* weights,
- const float* zero,
- float* output,
- uint32_t padding_top,
- const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
-{
- assert(input_height != 0);
- assert(input_width != 0);
- assert(input_width % sizeof(float) == 0);
- assert(padding_top >= 1);
- assert(padding_top <= 2);
-
- const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
- const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
- const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
- const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
-
- const float32x4_t vw0123 = vld1q_f32(weights);
- const float32x4_t vw4567 = vld1q_f32(weights + 4);
- const float32x4_t vw89AB = vld1q_f32(weights + 8);
- const float32x4_t vwCDEF = vld1q_f32(weights + 12);
- const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
- const float32x4_t vwKLMN = vld1q_f32(weights + 20);
- const float32x2_t vwOP = vld1_f32(weights + 24);
-
- const uint32_t padding_top_less_1 = padding_top - 1;
- const size_t input_decrement = round_down_po2(input_width - 1 * sizeof(float), 4 * sizeof(float)) + 4 * sizeof(float);
-
- const float* i0 = zero;
- const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
- const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
- if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
- i1 = zero;
- }
- const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
- const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
-
- size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
- size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
- do {
- if XNN_UNPREDICTABLE(padded_input_height <= 6) {
- i4 = zero;
- }
- if XNN_UNPREDICTABLE(padded_input_height < 6) {
- i3 = zero;
- }
-
- float32x4_t vi0x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi1x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi2x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi3x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi4x0123 = vmovq_n_f32(0.0f);
- float32x4_t vi0x4567 = vld1q_f32(i0); i0 += 4;
- float32x4_t vi1x4567 = vld1q_f32(i1); i1 += 4;
- float32x4_t vi2x4567 = vld1q_f32(i2); i2 += 4;
- float32x4_t vi3x4567 = vld1q_f32(i3); i3 += 4;
- float32x4_t vi4x4567 = vld1q_f32(i4); i4 += 4;
-
- size_t w = input_width;
- for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
- float32x4_t vo468Ap0 = vdupq_laneq_f32(vw0123, 0);
-
- float32x4_t vi0x89AB;
- float32x4_t vi1x89AB;
- float32x4_t vi2x89AB;
- float32x4_t vi3x89AB;
- float32x4_t vi4x89AB;
-
- vi0x89AB = vld1q_f32(i0); i0 += 4;
- vi1x89AB = vld1q_f32(i1); i1 += 4;
- vi2x89AB = vld1q_f32(i2); i2 += 4;
- vi3x89AB = vld1q_f32(i3); i3 += 4;
- vi4x89AB = vld1q_f32(i4); i4 += 4;
-
- float32x4_t vi0xCDEF;
- float32x4_t vi1xCDEF;
- float32x4_t vi2xCDEF;
- float32x4_t vi3xCDEF;
- float32x4_t vi4xCDEF;
-
- vi0xCDEF = vld1q_f32(i0); i0 += 4;
- vi1xCDEF = vld1q_f32(i1); i1 += 4;
- vi2xCDEF = vld1q_f32(i2); i2 += 4;
- vi3xCDEF = vld1q_f32(i3); i3 += 4;
- vi4xCDEF = vld1q_f32(i4); i4 += 4;
-
- float32x4_t vi0x468A = vuzp1q_f32(vi0x4567, vi0x89AB);
- float32x4_t vi0x579B = vuzp2q_f32(vi0x4567, vi0x89AB);
- float32x4_t vi1x468A = vuzp1q_f32(vi1x4567, vi1x89AB);
- float32x4_t vi1x579B = vuzp2q_f32(vi1x4567, vi1x89AB);
- float32x4_t vi2x468A = vuzp1q_f32(vi2x4567, vi2x89AB);
- float32x4_t vi2x579B = vuzp2q_f32(vi2x4567, vi2x89AB);
- float32x4_t vi3x468A = vuzp1q_f32(vi3x4567, vi3x89AB);
- float32x4_t vi3x579B = vuzp2q_f32(vi3x4567, vi3x89AB);
- float32x4_t vi4x468A = vuzp1q_f32(vi4x4567, vi4x89AB);
- float32x4_t vi4x579B = vuzp2q_f32(vi4x4567, vi4x89AB);
-
- // middle tap
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x468A, vw0123, 3);
- float32x4_t vo468Ap1 = vmulq_laneq_f32(vi1x468A, vw89AB, 0);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x468A, vwCDEF, 1);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x468A, vwGHIJ, 2);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi4x468A, vwKLMN, 3);
-
- // one left
- const float32x4_t vi0x3579 = vextq_f32(vi0x0123, vi0x579B, 3);
- const float32x4_t vi1x3579 = vextq_f32(vi1x0123, vi1x579B, 3);
- const float32x4_t vi2x3579 = vextq_f32(vi2x0123, vi2x579B, 3);
- const float32x4_t vi3x3579 = vextq_f32(vi3x0123, vi3x579B, 3);
- const float32x4_t vi4x3579 = vextq_f32(vi4x0123, vi4x579B, 3);
-
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x3579, vw0123, 2);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi1x3579, vw4567, 3);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x3579, vwCDEF, 0);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x3579, vwGHIJ, 1);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi4x3579, vwKLMN, 2);
-
- // two left
- // getting the vector to use for the far left tap is annoying
- // as we can't ext anything we currently have to get it.
- // To do this, we get a bit ugly. Interpret the float 32x4
- // vector as int 64x2. Then left shift by 32. Interpret
- // again as float 32x4. Now the right most bits are what we
- // want them to be for the following ext.
- const float32x4_t vi0x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi0x0123), 32));
- const float32x4_t vi1x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi1x0123), 32));
- const float32x4_t vi2x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi2x0123), 32));
- const float32x4_t vi3x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi3x0123), 32));
- const float32x4_t vi4x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi4x0123), 32));
-
- const float32x4_t vi0x2468 = vextq_f32(vi0x0012, vi0x468A, 3);
- const float32x4_t vi1x2468 = vextq_f32(vi1x0012, vi1x468A, 3);
- const float32x4_t vi2x2468 = vextq_f32(vi2x0012, vi2x468A, 3);
- const float32x4_t vi3x2468 = vextq_f32(vi3x0012, vi3x468A, 3);
- const float32x4_t vi4x2468 = vextq_f32(vi4x0012, vi4x468A, 3);
-
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x2468, vw0123, 1);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi1x2468, vw4567, 2);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x2468, vw89AB, 3);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x2468, vwGHIJ, 0);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi4x2468, vwKLMN, 1);
-
- vi0x0123 = vi0x89AB;
- vi1x0123 = vi1x89AB;
- vi2x0123 = vi2x89AB;
- vi3x0123 = vi3x89AB;
- vi4x0123 = vi4x89AB;
-
- // one right
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x579B, vw4567, 0);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi1x579B, vw89AB, 1);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x579B, vwCDEF, 2);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x579B, vwGHIJ, 3);
- vo468Ap0 = vfmaq_lane_f32( vo468Ap0, vi4x579B, vwOP, 0);
-
- // two right
- const float32x4_t vi0x68AC = vextq_f32(vi0x468A, vi0xCDEF, 1);
- const float32x4_t vi1x68AC = vextq_f32(vi1x468A, vi1xCDEF, 1);
- const float32x4_t vi2x68AC = vextq_f32(vi2x468A, vi2xCDEF, 1);
- const float32x4_t vi3x68AC = vextq_f32(vi3x468A, vi3xCDEF, 1);
- const float32x4_t vi4x68AC = vextq_f32(vi4x468A, vi4xCDEF, 1);
-
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x68AC, vw4567, 1);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi1x68AC, vw89AB, 2);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x68AC, vwCDEF, 3);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x68AC, vwKLMN, 0);
- vo468Ap0 = vfmaq_lane_f32( vo468Ap0, vi4x68AC, vwOP, 1);
-
- vi0x4567 = vi0xCDEF;
- vi1x4567 = vi1xCDEF;
- vi2x4567 = vi2xCDEF;
- vi3x4567 = vi3xCDEF;
- vi4x4567 = vi4xCDEF;
-
- float32x4_t vo0 = vaddq_f32(vo468Ap0, vo468Ap1);
-
- vo0 = vmaxq_f32(vo0, vmin);
- vo0 = vminq_f32(vo0, vmax);
-
- size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
- if XNN_LIKELY(w_tmp >= 4) {
- vst1q_f32(output, vo0); output += 4;
- } else {
- float* output_tmp = output;
- float32x2_t vo0_lo = vget_low_f32(vo0);
- if (w_tmp & 2) {
- vst1_f32(output_tmp, vo0_lo); output_tmp += 2;
- vo0_lo = vget_high_f32(vo0);
- }
- if (w_tmp & 1) {
- vst1_lane_f32(output_tmp, vo0_lo, 0);
- }
- }
- }
-
- {
- float32x4_t vo468Ap0 = vdupq_laneq_f32(vw0123, 0);
-
- float32x4_t vi0x89AB;
- float32x4_t vi1x89AB;
- float32x4_t vi2x89AB;
- float32x4_t vi3x89AB;
- float32x4_t vi4x89AB;
-
- if XNN_LIKELY(w > 4 * sizeof(float)) {
- vi0x89AB = vld1q_f32(i0); i0 += 4;
- vi1x89AB = vld1q_f32(i1); i1 += 4;
- vi2x89AB = vld1q_f32(i2); i2 += 4;
- vi3x89AB = vld1q_f32(i3); i3 += 4;
- vi4x89AB = vld1q_f32(i4); i4 += 4;
- } else {
- vi0x89AB = vmovq_n_f32(0.0f);
- vi1x89AB = vmovq_n_f32(0.0f);
- vi2x89AB = vmovq_n_f32(0.0f);
- vi3x89AB = vmovq_n_f32(0.0f);
- vi4x89AB = vmovq_n_f32(0.0f);
- }
-
- float32x4_t vi0xCDEF;
- float32x4_t vi1xCDEF;
- float32x4_t vi2xCDEF;
- float32x4_t vi3xCDEF;
- float32x4_t vi4xCDEF;
-
- if XNN_LIKELY(w > 8 * sizeof(float)) {
- vi0xCDEF = vld1q_f32(i0); i0 += 4;
- vi1xCDEF = vld1q_f32(i1); i1 += 4;
- vi2xCDEF = vld1q_f32(i2); i2 += 4;
- vi3xCDEF = vld1q_f32(i3); i3 += 4;
- vi4xCDEF = vld1q_f32(i4); i4 += 4;
- } else {
- vi0xCDEF = vmovq_n_f32(0.0f);
- vi1xCDEF = vmovq_n_f32(0.0f);
- vi2xCDEF = vmovq_n_f32(0.0f);
- vi3xCDEF = vmovq_n_f32(0.0f);
- vi4xCDEF = vmovq_n_f32(0.0f);
- }
- float32x4_t vi0x468A = vuzp1q_f32(vi0x4567, vi0x89AB);
- float32x4_t vi0x579B = vuzp2q_f32(vi0x4567, vi0x89AB);
- float32x4_t vi1x468A = vuzp1q_f32(vi1x4567, vi1x89AB);
- float32x4_t vi1x579B = vuzp2q_f32(vi1x4567, vi1x89AB);
- float32x4_t vi2x468A = vuzp1q_f32(vi2x4567, vi2x89AB);
- float32x4_t vi2x579B = vuzp2q_f32(vi2x4567, vi2x89AB);
- float32x4_t vi3x468A = vuzp1q_f32(vi3x4567, vi3x89AB);
- float32x4_t vi3x579B = vuzp2q_f32(vi3x4567, vi3x89AB);
- float32x4_t vi4x468A = vuzp1q_f32(vi4x4567, vi4x89AB);
- float32x4_t vi4x579B = vuzp2q_f32(vi4x4567, vi4x89AB);
-
- vi0x468A = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x468A)));
- vi1x468A = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x468A)));
- vi2x468A = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x468A)));
- vi3x468A = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x468A)));
- vi4x468A = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x468A)));
-
- vi0x579B = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x579B)));
- vi1x579B = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x579B)));
- vi2x579B = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x579B)));
- vi3x579B = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x579B)));
- vi4x579B = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x579B)));
-
- // middle tap
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x468A, vw0123, 3);
- float32x4_t vo468Ap1 = vmulq_laneq_f32(vi1x468A, vw89AB, 0);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x468A, vwCDEF, 1);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x468A, vwGHIJ, 2);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi4x468A, vwKLMN, 3);
-
- // one left
- const float32x4_t vi0x3579 = vextq_f32(vi0x0123, vi0x579B, 3);
- const float32x4_t vi1x3579 = vextq_f32(vi1x0123, vi1x579B, 3);
- const float32x4_t vi2x3579 = vextq_f32(vi2x0123, vi2x579B, 3);
- const float32x4_t vi3x3579 = vextq_f32(vi3x0123, vi3x579B, 3);
- const float32x4_t vi4x3579 = vextq_f32(vi4x0123, vi4x579B, 3);
-
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x3579, vw0123, 2);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi1x3579, vw4567, 3);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x3579, vwCDEF, 0);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x3579, vwGHIJ, 1);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi4x3579, vwKLMN, 2);
-
- // two left
- // getting the vector to use for the far left tap is annoying
- // as we can't ext anything we currently have to get it.
- // To do this, we get a bit ugly. Interpret the float 32x4
- // vector as int 64x2. Then left shift by 32. Interpret
- // again as float 32x4. Now the right most bits are what we
- // want them to be for the following ext.
- const float32x4_t vi0x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi0x0123), 32));
- const float32x4_t vi1x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi1x0123), 32));
- const float32x4_t vi2x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi2x0123), 32));
- const float32x4_t vi3x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi3x0123), 32));
- const float32x4_t vi4x0012 = vreinterpretq_f32_u64(vshlq_n_u64(vreinterpretq_u64_f32(vi4x0123), 32));
-
- const float32x4_t vi0x2468 = vextq_f32(vi0x0012, vi0x468A, 3);
- const float32x4_t vi1x2468 = vextq_f32(vi1x0012, vi1x468A, 3);
- const float32x4_t vi2x2468 = vextq_f32(vi2x0012, vi2x468A, 3);
- const float32x4_t vi3x2468 = vextq_f32(vi3x0012, vi3x468A, 3);
- const float32x4_t vi4x2468 = vextq_f32(vi4x0012, vi4x468A, 3);
-
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x2468, vw0123, 1);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi1x2468, vw4567, 2);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x2468, vw89AB, 3);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x2468, vwGHIJ, 0);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi4x2468, vwKLMN, 1);
-
- vi0x0123 = vi0x89AB;
- vi1x0123 = vi1x89AB;
- vi2x0123 = vi2x89AB;
- vi3x0123 = vi3x89AB;
- vi4x0123 = vi4x89AB;
-
- // one right
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x579B, vw4567, 0);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi1x579B, vw89AB, 1);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x579B, vwCDEF, 2);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x579B, vwGHIJ, 3);
- vo468Ap0 = vfmaq_lane_f32( vo468Ap0, vi4x579B, vwOP, 0);
-
- // two right
- const float32x4_t vi0x68AC = vextq_f32(vi0x468A, vi0xCDEF, 1);
- const float32x4_t vi1x68AC = vextq_f32(vi1x468A, vi1xCDEF, 1);
- const float32x4_t vi2x68AC = vextq_f32(vi2x468A, vi2xCDEF, 1);
- const float32x4_t vi3x68AC = vextq_f32(vi3x468A, vi3xCDEF, 1);
- const float32x4_t vi4x68AC = vextq_f32(vi4x468A, vi4xCDEF, 1);
-
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi0x68AC, vw4567, 1);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi1x68AC, vw89AB, 2);
- vo468Ap0 = vfmaq_laneq_f32(vo468Ap0, vi2x68AC, vwCDEF, 3);
- vo468Ap1 = vfmaq_laneq_f32(vo468Ap1, vi3x68AC, vwKLMN, 0);
- vo468Ap0 = vfmaq_lane_f32( vo468Ap0, vi4x68AC, vwOP, 1);
-
- vi0x4567 = vi0xCDEF;
- vi1x4567 = vi1xCDEF;
- vi2x4567 = vi2xCDEF;
- vi3x4567 = vi3xCDEF;
- vi4x4567 = vi4xCDEF;
-
- float32x4_t vo0 = vaddq_f32(vo468Ap0, vo468Ap1);
-
- vo0 = vmaxq_f32(vo0, vmin);
- vo0 = vminq_f32(vo0, vmax);
-
- size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
- if XNN_LIKELY(w_tmp >= 4) {
- vst1q_f32(output, vo0); output += 4;
- } else {
- float32x2_t vo0_lo = vget_low_f32(vo0);
- if (w_tmp & 2) {
- vst1_f32(output, vo0_lo); output += 2;
- vo0_lo = vget_high_f32(vo0);
- }
- if (w_tmp & 1) {
- vst1_lane_f32(output, vo0_lo, 0); output += 1;
- }
- }
- }
-
- i0 = (const float*) ((uintptr_t) i2 - input_decrement);
- i1 = (const float*) ((uintptr_t) i3 - input_decrement);
- i2 = (const float*) ((uintptr_t) i4 - input_decrement);
- i3 = (const float*) ((uintptr_t) i2 + input_width);
- i4 = (const float*) ((uintptr_t) i3 + input_width);
-
- output_height -= 1;
- padded_input_height -= 2;
- } while (output_height != 0);
-}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc2.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc2.c
new file mode 100644
index 0000000..265039e
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc2.c
@@ -0,0 +1,310 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc3.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc3.c
new file mode 100644
index 0000000..8d43087
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc3.c
@@ -0,0 +1,312 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc4.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc4.c
new file mode 100644
index 0000000..ad4e95b
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc4.c
@@ -0,0 +1,314 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ float32x4_t vo0p3 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p2 = vaddq_f32(vo0p2, vo0p3);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ float32x4_t vo0p3 = vmulq_lane_f32(vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p2 = vaddq_f32(vo0p2, vo0p3);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc5.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc5.c
new file mode 100644
index 0000000..b55fa4e
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4-acc5.c
@@ -0,0 +1,316 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ float32x4_t vo0p3 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ float32x4_t vo0p4 = vmulq_lane_f32(vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p4 = vmlaq_lane_f32(vo0p4, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p4 = vmlaq_lane_f32(vo0p4, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p4 = vmlaq_lane_f32(vo0p4, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p4 = vmlaq_lane_f32(vo0p4, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p2 = vaddq_f32(vo0p2, vo0p3);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+ vo0p0 = vaddq_f32(vo0p0, vo0p4);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ float32x4_t vo0p3 = vmulq_lane_f32(vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ float32x4_t vo0p4 = vmulq_lane_f32(vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p4 = vmlaq_lane_f32(vo0p4, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p4 = vmlaq_lane_f32(vo0p4, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p4 = vmlaq_lane_f32(vo0p4, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p3 = vmlaq_lane_f32(vo0p3, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p4 = vmlaq_lane_f32(vo0p4, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p2 = vaddq_f32(vo0p2, vo0p3);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+ vo0p0 = vaddq_f32(vo0p0, vo0p4);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4.c
new file mode 100644
index 0000000..d5b374f
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-1x4.c
@@ -0,0 +1,308 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc2.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc2.c
new file mode 100644
index 0000000..dfc5349
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc2.c
@@ -0,0 +1,419 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x9BDF, vwOP, 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ if (w_tmp & 2) {
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i5 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ o0 = o1;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ output_height = doz(output_height, 2);
+ padded_input_height = doz(padded_input_height, 4);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc3.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc3.c
new file mode 100644
index 0000000..132f05b
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4-acc3.c
@@ -0,0 +1,423 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ float32x4_t vo1p2 = vmulq_lane_f32(vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi6x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi6xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+ vo1p0 = vaddq_f32(vo1p0, vo1p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE, vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE, vget_low_f32(vw89AB), 0);
+ float32x4_t vo1p2 = vmulq_lane_f32(vi3x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi3x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4x9BDF, vwOP, 0);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi6x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vmlaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+ vo1p2 = vmlaq_lane_f32(vo1p2, vi6xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+ vo1p0 = vaddq_f32(vo1p0, vo1p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ if (w_tmp & 2) {
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i5 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ o0 = o1;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ output_height = doz(output_height, 2);
+ padded_input_height = doz(padded_input_height, 4);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4.c
new file mode 100644
index 0000000..8620739
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-2x4.c
@@ -0,0 +1,415 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE, vget_high_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x9BDF, vwOP, 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ if (w_tmp & 2) {
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i5 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ o0 = o1;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ output_height = doz(output_height, 2);
+ padded_input_height = doz(padded_input_height, 4);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4-acc2.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4-acc2.c
new file mode 100644
index 0000000..e737a16
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4-acc2.c
@@ -0,0 +1,526 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+ const float* i7 = (const float*) ((uintptr_t) i6 + input_width);
+ const float* i8 = (const float*) ((uintptr_t) i7 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+ float* o2 = (float*) ((uintptr_t) o1 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ o2 = o1;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 10) {
+ i7 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 11) {
+ i8 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi7x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi8x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi7x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi8x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+ float32x4x2_t vi7x8ACE9BDF = vld2q_f32(i7); i7 += 8;
+ float32x4x2_t vi8x8ACE9BDF = vld2q_f32(i8); i8 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo2p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ float32x4_t vo2p1 = vmulq_lane_f32(vi4x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi7x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi4x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi6x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x8ACE9BDF.val[1], vwOP, 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi8x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+ const float32x4_t vi7x68AC = vextq_f32(vi7x0246, vi7x8ACE9BDF.val[0], 3);
+ vi7x0246 = vi7x8ACE9BDF.val[0];
+ const float32x4_t vi8x68AC = vextq_f32(vi8x0246, vi8x8ACE9BDF.val[0], 3);
+ vi8x0246 = vi8x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi5x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi7x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+ const float32x4_t vi7x79BD = vextq_f32(vi7x1357, vi7x8ACE9BDF.val[1], 3);
+ vi7x1357 = vi7x8ACE9BDF.val[1];
+ const float32x4_t vi8x79BD = vextq_f32(vi8x1357, vi8x8ACE9BDF.val[1], 3);
+ vi8x1357 = vi8x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+ const float32x4x2_t vi7xGIKMHJLN = vld2q_f32(i7); i7 += 8;
+ const float32x4x2_t vi8xGIKMHJLN = vld2q_f32(i8); i8 += 8;
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x79BD, vget_high_f32(vw0123), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi4x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi6x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x79BD, vget_high_f32(vwKLMN), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi8x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+ const float32x4_t vi7xACEG = vextq_f32(vi7x8ACE9BDF.val[0], vi7xGIKMHJLN.val[0], 1);
+ vi7x8ACE9BDF = vi7xGIKMHJLN;
+ const float32x4_t vi8xACEG = vextq_f32(vi8x8ACE9BDF.val[0], vi8xGIKMHJLN.val[0], 1);
+ vi8x8ACE9BDF = vi8xGIKMHJLN;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3xACEG, vget_high_f32(vw89AB), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi5xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi7xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+ vo2p0 = vaddq_f32(vo2p0, vo2p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+ float32x4_t vo2 = vmaxq_f32(vo2p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+ vo2 = vminq_f32(vo2, vmax);
+
+ vst1q_f32(o2, vo2); o2 += 4;
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo2p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+ const float32x4_t vi7x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi7x8ACE9BDF.val[0])));
+ const float32x4_t vi8x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi8x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+ const float32x4_t vi7x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi7x8ACE9BDF.val[1])));
+ const float32x4_t vi8x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi8x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE, vget_high_f32(vw0123), 1);
+ float32x4_t vo2p1 = vmulq_lane_f32(vi4x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE, vget_low_f32(vw89AB), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi7x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x9BDF, vget_low_f32(vw4567), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi4x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x9BDF, vget_low_f32(vw89AB), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi6x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x9BDF, vwOP, 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x9BDF, vwOP, 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi8x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+ const float32x4_t vi7x68AC = vextq_f32(vi7x0246, vi7x8ACE, 3);
+ const float32x4_t vi8x68AC = vextq_f32(vi8x0246, vi8x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi5x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi7x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+ const float32x4_t vi7x79BD = vextq_f32(vi7x1357, vi7x9BDF, 3);
+ const float32x4_t vi8x79BD = vextq_f32(vi8x1357, vi8x9BDF, 3);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi2x79BD, vget_high_f32(vw0123), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi4x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi6x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi6x79BD, vget_high_f32(vwKLMN), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi8x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+ const float32x4_t vi7xACEG = vextq_f32(vi7x8ACE, vzero, 1);
+ const float32x4_t vi8xACEG = vextq_f32(vi8x8ACE, vzero, 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi3xACEG, vget_high_f32(vw89AB), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi5xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vmlaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vmlaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+ vo2p1 = vmlaq_lane_f32(vo2p1, vi7xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+ vo2p0 = vaddq_f32(vo2p0, vo2p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+ float32x4_t vo2 = vmaxq_f32(vo2p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+ vo2 = vminq_f32(vo2, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o2, vo2); o2 += 4;
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ float32x2_t vo2_lo = vget_low_f32(vo2);
+ if (w_tmp & 2) {
+ vst1_f32(o2, vo2_lo); o2 += 2;
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ vo2_lo = vget_high_f32(vo2);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o2, vo2_lo, 0); o2 += 1;
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i7 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i8 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+ i7 = (const float*) ((uintptr_t) i6 + input_width);
+ i8 = (const float*) ((uintptr_t) i7 + input_width);
+
+ o0 = o2;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+ o2 = (float*) ((uintptr_t) o1 + output_width);
+
+ output_height = doz(output_height, 3);
+ padded_input_height = doz(padded_input_height, 6);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4.c
new file mode 100644
index 0000000..2ebae1a
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neon-3x4.c
@@ -0,0 +1,520 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+ const float* i7 = (const float*) ((uintptr_t) i6 + input_width);
+ const float* i8 = (const float*) ((uintptr_t) i7 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+ float* o2 = (float*) ((uintptr_t) o1 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ o2 = o1;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 10) {
+ i7 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 11) {
+ i8 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi7x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi8x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi7x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi8x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+ float32x4x2_t vi7x8ACE9BDF = vld2q_f32(i7); i7 += 8;
+ float32x4x2_t vi8x8ACE9BDF = vld2q_f32(i8); i8 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo2p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[1], vwOP, 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+ const float32x4_t vi7x68AC = vextq_f32(vi7x0246, vi7x8ACE9BDF.val[0], 3);
+ vi7x0246 = vi7x8ACE9BDF.val[0];
+ const float32x4_t vi8x68AC = vextq_f32(vi8x0246, vi8x8ACE9BDF.val[0], 3);
+ vi8x0246 = vi8x8ACE9BDF.val[0];
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x68AC, vget_high_f32(vw4567), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+ const float32x4_t vi7x79BD = vextq_f32(vi7x1357, vi7x8ACE9BDF.val[1], 3);
+ vi7x1357 = vi7x8ACE9BDF.val[1];
+ const float32x4_t vi8x79BD = vextq_f32(vi8x1357, vi8x8ACE9BDF.val[1], 3);
+ vi8x1357 = vi8x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+ const float32x4x2_t vi7xGIKMHJLN = vld2q_f32(i7); i7 += 8;
+ const float32x4x2_t vi8xGIKMHJLN = vld2q_f32(i8); i8 += 8;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x79BD, vget_high_f32(vw0123), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x79BD, vget_low_f32(vwCDEF), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+ const float32x4_t vi7xACEG = vextq_f32(vi7x8ACE9BDF.val[0], vi7xGIKMHJLN.val[0], 1);
+ vi7x8ACE9BDF = vi7xGIKMHJLN;
+ const float32x4_t vi8xACEG = vextq_f32(vi8x8ACE9BDF.val[0], vi8xGIKMHJLN.val[0], 1);
+ vi8x8ACE9BDF = vi8xGIKMHJLN;
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3xACEG, vget_high_f32(vw89AB), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5xACEG, vget_low_f32(vwKLMN), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+ float32x4_t vo2 = vmaxq_f32(vo2p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+ vo2 = vminq_f32(vo2, vmax);
+
+ vst1q_f32(o2, vo2); o2 += 4;
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo2p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+ const float32x4_t vi7x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi7x8ACE9BDF.val[0])));
+ const float32x4_t vi8x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi8x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+ const float32x4_t vi7x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi7x8ACE9BDF.val[1])));
+ const float32x4_t vi8x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi8x8ACE9BDF.val[1])));
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x8ACE, vget_high_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x8ACE, vget_high_f32(vw0123), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x8ACE, vget_low_f32(vw89AB), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x9BDF, vget_low_f32(vw4567), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x9BDF, vget_low_f32(vw89AB), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x9BDF, vwOP, 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x9BDF, vwOP, 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+ const float32x4_t vi7x68AC = vextq_f32(vi7x0246, vi7x8ACE, 3);
+ const float32x4_t vi8x68AC = vextq_f32(vi8x0246, vi8x8ACE, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x68AC, vget_high_f32(vw4567), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+ const float32x4_t vi7x79BD = vextq_f32(vi7x1357, vi7x9BDF, 3);
+ const float32x4_t vi8x79BD = vextq_f32(vi8x1357, vi8x9BDF, 3);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2x79BD, vget_high_f32(vw0123), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4x79BD, vget_low_f32(vwCDEF), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+ const float32x4_t vi7xACEG = vextq_f32(vi7x8ACE, vzero, 1);
+ const float32x4_t vi8xACEG = vextq_f32(vi8x8ACE, vzero, 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi4xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi3xACEG, vget_high_f32(vw89AB), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi5xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi6xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi5xACEG, vget_low_f32(vwKLMN), 0);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi7xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vmlaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vmlaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+ vo2p0 = vmlaq_lane_f32(vo2p0, vi8xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+ float32x4_t vo2 = vmaxq_f32(vo2p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+ vo2 = vminq_f32(vo2, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o2, vo2); o2 += 4;
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ float32x2_t vo2_lo = vget_low_f32(vo2);
+ if (w_tmp & 2) {
+ vst1_f32(o2, vo2_lo); o2 += 2;
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ vo2_lo = vget_high_f32(vo2);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o2, vo2_lo, 0); o2 += 1;
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i7 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i8 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+ i7 = (const float*) ((uintptr_t) i6 + input_width);
+ i8 = (const float*) ((uintptr_t) i7 + input_width);
+
+ o0 = o2;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+ o2 = (float*) ((uintptr_t) o1 + output_width);
+
+ output_height = doz(output_height, 3);
+ padded_input_height = doz(padded_input_height, 6);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc2.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc2.c
new file mode 100644
index 0000000..2057bd9
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc2.c
@@ -0,0 +1,310 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc2(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc3.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc3.c
new file mode 100644
index 0000000..6f80884
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc3.c
@@ -0,0 +1,312 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc4.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc4.c
new file mode 100644
index 0000000..ab67981
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc4.c
@@ -0,0 +1,314 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ float32x4_t vo0p3 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p2 = vaddq_f32(vo0p2, vo0p3);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ float32x4_t vo0p3 = vmulq_lane_f32(vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p2 = vaddq_f32(vo0p2, vo0p3);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc5.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc5.c
new file mode 100644
index 0000000..ce11525
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4-acc5.c
@@ -0,0 +1,316 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ float32x4_t vo0p3 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ float32x4_t vo0p4 = vmulq_lane_f32(vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p4 = vfmaq_lane_f32(vo0p4, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p4 = vfmaq_lane_f32(vo0p4, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p4 = vfmaq_lane_f32(vo0p4, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p4 = vfmaq_lane_f32(vo0p4, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p2 = vaddq_f32(vo0p2, vo0p3);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+ vo0p0 = vaddq_f32(vo0p0, vo0p4);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ float32x4_t vo0p3 = vmulq_lane_f32(vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ float32x4_t vo0p4 = vmulq_lane_f32(vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p4 = vfmaq_lane_f32(vo0p4, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p4 = vfmaq_lane_f32(vo0p4, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p4 = vfmaq_lane_f32(vo0p4, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p3 = vfmaq_lane_f32(vo0p3, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p4 = vfmaq_lane_f32(vo0p4, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo0p2 = vaddq_f32(vo0p2, vo0p3);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+ vo0p0 = vaddq_f32(vo0p0, vo0p4);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4.c
new file mode 100644
index 0000000..11b2e5f
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-1x4.c
@@ -0,0 +1,308 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ float* o0 = output;
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ if (w_tmp & 2) {
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i2 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i3 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+
+
+ output_height -= 1;
+ padded_input_height -= 2;
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc2.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc2.c
new file mode 100644
index 0000000..48bcb70
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc2.c
@@ -0,0 +1,419 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x9BDF, vwOP, 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ if (w_tmp & 2) {
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i5 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ o0 = o1;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ output_height = doz(output_height, 2);
+ padded_input_height = doz(padded_input_height, 4);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc3.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc3.c
new file mode 100644
index 0000000..68eb42f
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4-acc3.c
@@ -0,0 +1,423 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ float32x4_t vo1p2 = vmulq_lane_f32(vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi6x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi6xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+ vo1p0 = vaddq_f32(vo1p0, vo1p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE, vget_high_f32(vw0123), 1);
+
+ float32x4_t vo0p2 = vmulq_lane_f32(vi1x8ACE, vget_low_f32(vw89AB), 0);
+ float32x4_t vo1p2 = vmulq_lane_f32(vi3x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi3x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4x9BDF, vwOP, 0);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi6x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p2 = vfmaq_lane_f32(vo0p2, vi4xACEG, vwOP, 1);
+ vo1p2 = vfmaq_lane_f32(vo1p2, vi6xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+ vo0p0 = vaddq_f32(vo0p0, vo0p2);
+ vo1p0 = vaddq_f32(vo1p0, vo1p2);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ if (w_tmp & 2) {
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i5 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ o0 = o1;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ output_height = doz(output_height, 2);
+ padded_input_height = doz(padded_input_height, 4);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4.c
new file mode 100644
index 0000000..db4175f
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-2x4.c
@@ -0,0 +1,415 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE, vget_high_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x9BDF, vwOP, 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ if (w_tmp & 2) {
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i4 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i5 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+
+ o0 = o1;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+
+ output_height = doz(output_height, 2);
+ padded_input_height = doz(padded_input_height, 4);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4-acc2.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4-acc2.c
new file mode 100644
index 0000000..303f99f
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4-acc2.c
@@ -0,0 +1,526 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+ const float* i7 = (const float*) ((uintptr_t) i6 + input_width);
+ const float* i8 = (const float*) ((uintptr_t) i7 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+ float* o2 = (float*) ((uintptr_t) o1 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ o2 = o1;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 10) {
+ i7 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 11) {
+ i8 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi7x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi8x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi7x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi8x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+ float32x4x2_t vi7x8ACE9BDF = vld2q_f32(i7); i7 += 8;
+ float32x4x2_t vi8x8ACE9BDF = vld2q_f32(i8); i8 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo2p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ float32x4_t vo2p1 = vmulq_lane_f32(vi4x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi7x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi4x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi6x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x8ACE9BDF.val[1], vwOP, 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi8x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+ const float32x4_t vi7x68AC = vextq_f32(vi7x0246, vi7x8ACE9BDF.val[0], 3);
+ vi7x0246 = vi7x8ACE9BDF.val[0];
+ const float32x4_t vi8x68AC = vextq_f32(vi8x0246, vi8x8ACE9BDF.val[0], 3);
+ vi8x0246 = vi8x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi5x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi7x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+ const float32x4_t vi7x79BD = vextq_f32(vi7x1357, vi7x8ACE9BDF.val[1], 3);
+ vi7x1357 = vi7x8ACE9BDF.val[1];
+ const float32x4_t vi8x79BD = vextq_f32(vi8x1357, vi8x8ACE9BDF.val[1], 3);
+ vi8x1357 = vi8x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+ const float32x4x2_t vi7xGIKMHJLN = vld2q_f32(i7); i7 += 8;
+ const float32x4x2_t vi8xGIKMHJLN = vld2q_f32(i8); i8 += 8;
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x79BD, vget_high_f32(vw0123), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi4x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi6x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x79BD, vget_high_f32(vwKLMN), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi8x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+ const float32x4_t vi7xACEG = vextq_f32(vi7x8ACE9BDF.val[0], vi7xGIKMHJLN.val[0], 1);
+ vi7x8ACE9BDF = vi7xGIKMHJLN;
+ const float32x4_t vi8xACEG = vextq_f32(vi8x8ACE9BDF.val[0], vi8xGIKMHJLN.val[0], 1);
+ vi8x8ACE9BDF = vi8xGIKMHJLN;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3xACEG, vget_high_f32(vw89AB), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi5xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi7xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+ vo2p0 = vaddq_f32(vo2p0, vo2p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+ float32x4_t vo2 = vmaxq_f32(vo2p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+ vo2 = vminq_f32(vo2, vmax);
+
+ vst1q_f32(o2, vo2); o2 += 4;
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo2p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+ const float32x4_t vi7x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi7x8ACE9BDF.val[0])));
+ const float32x4_t vi8x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi8x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+ const float32x4_t vi7x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi7x8ACE9BDF.val[1])));
+ const float32x4_t vi8x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi8x8ACE9BDF.val[1])));
+
+ float32x4_t vo0p1 = vmulq_lane_f32(vi0x8ACE, vget_high_f32(vw0123), 1);
+ float32x4_t vo1p1 = vmulq_lane_f32(vi2x8ACE, vget_high_f32(vw0123), 1);
+ float32x4_t vo2p1 = vmulq_lane_f32(vi4x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE, vget_low_f32(vw89AB), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi7x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x9BDF, vget_low_f32(vw4567), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi4x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x9BDF, vget_low_f32(vw89AB), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi6x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x9BDF, vwOP, 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x9BDF, vwOP, 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi8x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+ const float32x4_t vi7x68AC = vextq_f32(vi7x0246, vi7x8ACE, 3);
+ const float32x4_t vi8x68AC = vextq_f32(vi8x0246, vi8x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3x68AC, vget_high_f32(vw4567), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi5x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi7x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+ const float32x4_t vi7x79BD = vextq_f32(vi7x1357, vi7x9BDF, 3);
+ const float32x4_t vi8x79BD = vextq_f32(vi8x1357, vi8x9BDF, 3);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi2x79BD, vget_high_f32(vw0123), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi4x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi4x79BD, vget_low_f32(vwCDEF), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi6x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi6x79BD, vget_high_f32(vwKLMN), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi8x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+ const float32x4_t vi7xACEG = vextq_f32(vi7x8ACE, vzero, 1);
+ const float32x4_t vi8xACEG = vextq_f32(vi8x8ACE, vzero, 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi3xACEG, vget_high_f32(vw89AB), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi5xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p1 = vfmaq_lane_f32(vo0p1, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p1 = vfmaq_lane_f32(vo1p1, vi5xACEG, vget_low_f32(vwKLMN), 0);
+ vo2p1 = vfmaq_lane_f32(vo2p1, vi7xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8xACEG, vwOP, 1);
+
+ vo0p0 = vaddq_f32(vo0p0, vo0p1);
+ vo1p0 = vaddq_f32(vo1p0, vo1p1);
+ vo2p0 = vaddq_f32(vo2p0, vo2p1);
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+ float32x4_t vo2 = vmaxq_f32(vo2p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+ vo2 = vminq_f32(vo2, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o2, vo2); o2 += 4;
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ float32x2_t vo2_lo = vget_low_f32(vo2);
+ if (w_tmp & 2) {
+ vst1_f32(o2, vo2_lo); o2 += 2;
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ vo2_lo = vget_high_f32(vo2);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o2, vo2_lo, 0); o2 += 1;
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i7 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i8 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+ i7 = (const float*) ((uintptr_t) i6 + input_width);
+ i8 = (const float*) ((uintptr_t) i7 + input_width);
+
+ o0 = o2;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+ o2 = (float*) ((uintptr_t) o1 + output_width);
+
+ output_height = doz(output_height, 3);
+ padded_input_height = doz(padded_input_height, 6);
+ } while (output_height != 0);
+}
diff --git a/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4.c b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4.c
new file mode 100644
index 0000000..661bc68
--- /dev/null
+++ b/src/f32-dwconv2d-chw/gen/5x5s2p2-minmax-neonfma-3x4.c
@@ -0,0 +1,520 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-dwconv2d-chw/5x5s2p2-neon.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/math.h>
+
+
+void xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4(
+ size_t input_height,
+ size_t input_width,
+ const float* input,
+ const float* weights,
+ const float* zero,
+ float* output,
+ uint32_t padding_top,
+ const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)])
+{
+ assert(input_height != 0);
+ assert(input_width != 0);
+ assert(input_width % sizeof(float) == 0);
+ assert(padding_top >= 1);
+ assert(padding_top <= 2);
+
+ const uint32x4_t vmask_even = vld1q_u32(params->neon.mask_even);
+ const uint32x4_t vmask_odd = vld1q_u32(params->neon.mask_odd);
+ const float32x4_t vmax = vld1q_dup_f32(¶ms->neon.max);
+ const float32x4_t vmin = vld1q_dup_f32(¶ms->neon.min);
+
+ const float32x4_t vw0123 = vld1q_f32(weights);
+ const float32x4_t vw4567 = vld1q_f32(weights + 4);
+ const float32x4_t vw89AB = vld1q_f32(weights + 8);
+ const float32x4_t vwCDEF = vld1q_f32(weights + 12);
+ const float32x4_t vwGHIJ = vld1q_f32(weights + 16);
+ const float32x4_t vwKLMN = vld1q_f32(weights + 20);
+ const float32x2_t vwOP = vld1_f32(weights + 24);
+
+ const uint32_t padding_top_less_1 = padding_top - 1;
+ const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(float));
+
+ const float* i0 = zero;
+ const float* i1 = (const float*) ((uintptr_t) input - ((-padding_top_less_1) & input_width));
+ const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
+ if XNN_UNPREDICTABLE(padding_top_less_1 != 0) {
+ i1 = zero;
+ }
+ const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
+ const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
+ const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
+ const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
+ const float* i7 = (const float*) ((uintptr_t) i6 + input_width);
+ const float* i8 = (const float*) ((uintptr_t) i7 + input_width);
+
+ const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
+
+ float* o0 = output;
+ float* o1 = (float*) ((uintptr_t) o0 + output_width);
+ float* o2 = (float*) ((uintptr_t) o1 + output_width);
+
+ size_t padded_input_height = input_height + (padding_top_less_1 + 1) + 2 /* padding bottom */;
+ size_t output_height = (padded_input_height - 5 /* kernel size */ + 2 /* subsampling */) / 2;
+ do {
+ if XNN_UNPREDICTABLE(padded_input_height < 6) {
+ i3 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 7) {
+ i4 = zero;
+ o1 = o0;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 8) {
+ i5 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 9) {
+ i6 = zero;
+ o2 = o1;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 10) {
+ i7 = zero;
+ }
+ if XNN_UNPREDICTABLE(padded_input_height < 11) {
+ i8 = zero;
+ }
+
+ float32x4_t vi0x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi7x0246 = vmovq_n_f32(0.0f);
+ float32x4_t vi8x0246 = vmovq_n_f32(0.0f);
+
+ float32x4_t vi0x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi1x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi2x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi3x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi4x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi5x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi6x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi7x1357 = vmovq_n_f32(0.0f);
+ float32x4_t vi8x1357 = vmovq_n_f32(0.0f);
+
+ float32x4x2_t vi0x8ACE9BDF = vld2q_f32(i0); i0 += 8;
+ float32x4x2_t vi1x8ACE9BDF = vld2q_f32(i1); i1 += 8;
+ float32x4x2_t vi2x8ACE9BDF = vld2q_f32(i2); i2 += 8;
+ float32x4x2_t vi3x8ACE9BDF = vld2q_f32(i3); i3 += 8;
+ float32x4x2_t vi4x8ACE9BDF = vld2q_f32(i4); i4 += 8;
+ float32x4x2_t vi5x8ACE9BDF = vld2q_f32(i5); i5 += 8;
+ float32x4x2_t vi6x8ACE9BDF = vld2q_f32(i6); i6 += 8;
+ float32x4x2_t vi7x8ACE9BDF = vld2q_f32(i7); i7 += 8;
+ float32x4x2_t vi8x8ACE9BDF = vld2q_f32(i8); i8 += 8;
+
+ size_t w = input_width;
+ for (; w > 8 * sizeof(float); w -= 8 * sizeof(float)) {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo2p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x8ACE9BDF.val[0], vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x8ACE9BDF.val[0], vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x8ACE9BDF.val[0], vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x8ACE9BDF.val[0], vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x8ACE9BDF.val[0], vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x8ACE9BDF.val[1], vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x8ACE9BDF.val[1], vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x8ACE9BDF.val[1], vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x8ACE9BDF.val[1], vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE9BDF.val[1], vwOP, 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE9BDF.val[1], vwOP, 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x8ACE9BDF.val[1], vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE9BDF.val[0], 3);
+ vi0x0246 = vi0x8ACE9BDF.val[0];
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE9BDF.val[0], 3);
+ vi1x0246 = vi1x8ACE9BDF.val[0];
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE9BDF.val[0], 3);
+ vi2x0246 = vi2x8ACE9BDF.val[0];
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE9BDF.val[0], 3);
+ vi3x0246 = vi3x8ACE9BDF.val[0];
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE9BDF.val[0], 3);
+ vi4x0246 = vi4x8ACE9BDF.val[0];
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE9BDF.val[0], 3);
+ vi5x0246 = vi5x8ACE9BDF.val[0];
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE9BDF.val[0], 3);
+ vi6x0246 = vi6x8ACE9BDF.val[0];
+ const float32x4_t vi7x68AC = vextq_f32(vi7x0246, vi7x8ACE9BDF.val[0], 3);
+ vi7x0246 = vi7x8ACE9BDF.val[0];
+ const float32x4_t vi8x68AC = vextq_f32(vi8x0246, vi8x8ACE9BDF.val[0], 3);
+ vi8x0246 = vi8x8ACE9BDF.val[0];
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x68AC, vget_high_f32(vw4567), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x8ACE9BDF.val[1], 3);
+ vi0x1357 = vi0x8ACE9BDF.val[1];
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x8ACE9BDF.val[1], 3);
+ vi1x1357 = vi1x8ACE9BDF.val[1];
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x8ACE9BDF.val[1], 3);
+ vi2x1357 = vi2x8ACE9BDF.val[1];
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x8ACE9BDF.val[1], 3);
+ vi3x1357 = vi3x8ACE9BDF.val[1];
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x8ACE9BDF.val[1], 3);
+ vi4x1357 = vi4x8ACE9BDF.val[1];
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x8ACE9BDF.val[1], 3);
+ vi5x1357 = vi5x8ACE9BDF.val[1];
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x8ACE9BDF.val[1], 3);
+ vi6x1357 = vi6x8ACE9BDF.val[1];
+ const float32x4_t vi7x79BD = vextq_f32(vi7x1357, vi7x8ACE9BDF.val[1], 3);
+ vi7x1357 = vi7x8ACE9BDF.val[1];
+ const float32x4_t vi8x79BD = vextq_f32(vi8x1357, vi8x8ACE9BDF.val[1], 3);
+ vi8x1357 = vi8x8ACE9BDF.val[1];
+
+ const float32x4x2_t vi0xGIKMHJLN = vld2q_f32(i0); i0 += 8;
+ const float32x4x2_t vi1xGIKMHJLN = vld2q_f32(i1); i1 += 8;
+ const float32x4x2_t vi2xGIKMHJLN = vld2q_f32(i2); i2 += 8;
+ const float32x4x2_t vi3xGIKMHJLN = vld2q_f32(i3); i3 += 8;
+ const float32x4x2_t vi4xGIKMHJLN = vld2q_f32(i4); i4 += 8;
+ const float32x4x2_t vi5xGIKMHJLN = vld2q_f32(i5); i5 += 8;
+ const float32x4x2_t vi6xGIKMHJLN = vld2q_f32(i6); i6 += 8;
+ const float32x4x2_t vi7xGIKMHJLN = vld2q_f32(i7); i7 += 8;
+ const float32x4x2_t vi8xGIKMHJLN = vld2q_f32(i8); i8 += 8;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x79BD, vget_high_f32(vw0123), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x79BD, vget_low_f32(vwCDEF), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE9BDF.val[0], vi0xGIKMHJLN.val[0], 1);
+ vi0x8ACE9BDF = vi0xGIKMHJLN;
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE9BDF.val[0], vi1xGIKMHJLN.val[0], 1);
+ vi1x8ACE9BDF = vi1xGIKMHJLN;
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE9BDF.val[0], vi2xGIKMHJLN.val[0], 1);
+ vi2x8ACE9BDF = vi2xGIKMHJLN;
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE9BDF.val[0], vi3xGIKMHJLN.val[0], 1);
+ vi3x8ACE9BDF = vi3xGIKMHJLN;
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE9BDF.val[0], vi4xGIKMHJLN.val[0], 1);
+ vi4x8ACE9BDF = vi4xGIKMHJLN;
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE9BDF.val[0], vi5xGIKMHJLN.val[0], 1);
+ vi5x8ACE9BDF = vi5xGIKMHJLN;
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE9BDF.val[0], vi6xGIKMHJLN.val[0], 1);
+ vi6x8ACE9BDF = vi6xGIKMHJLN;
+ const float32x4_t vi7xACEG = vextq_f32(vi7x8ACE9BDF.val[0], vi7xGIKMHJLN.val[0], 1);
+ vi7x8ACE9BDF = vi7xGIKMHJLN;
+ const float32x4_t vi8xACEG = vextq_f32(vi8x8ACE9BDF.val[0], vi8xGIKMHJLN.val[0], 1);
+ vi8x8ACE9BDF = vi8xGIKMHJLN;
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3xACEG, vget_high_f32(vw89AB), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5xACEG, vget_low_f32(vwKLMN), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+ float32x4_t vo2 = vmaxq_f32(vo2p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+ vo2 = vminq_f32(vo2, vmax);
+
+ vst1q_f32(o2, vo2); o2 += 4;
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ }
+ // Last block has 1-8 pixels to process.
+ assert(w <= 8 * sizeof(float));
+ assert(w >= 1 * sizeof(float));
+ {
+ float32x4_t vo0p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo1p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+ float32x4_t vo2p0 = vdupq_lane_f32(vget_low_f32(vw0123), 0);
+
+ const float32x4_t vi0x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[0])));
+ const float32x4_t vi1x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[0])));
+ const float32x4_t vi2x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[0])));
+ const float32x4_t vi3x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[0])));
+ const float32x4_t vi4x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[0])));
+ const float32x4_t vi5x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[0])));
+ const float32x4_t vi6x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[0])));
+ const float32x4_t vi7x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi7x8ACE9BDF.val[0])));
+ const float32x4_t vi8x8ACE = vreinterpretq_f32_u32(vandq_u32(vmask_even, vreinterpretq_u32_f32(vi8x8ACE9BDF.val[0])));
+
+ const float32x4_t vi0x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi0x8ACE9BDF.val[1])));
+ const float32x4_t vi1x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi1x8ACE9BDF.val[1])));
+ const float32x4_t vi2x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi2x8ACE9BDF.val[1])));
+ const float32x4_t vi3x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi3x8ACE9BDF.val[1])));
+ const float32x4_t vi4x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi4x8ACE9BDF.val[1])));
+ const float32x4_t vi5x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi5x8ACE9BDF.val[1])));
+ const float32x4_t vi6x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi6x8ACE9BDF.val[1])));
+ const float32x4_t vi7x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi7x8ACE9BDF.val[1])));
+ const float32x4_t vi8x9BDF = vreinterpretq_f32_u32(vandq_u32(vmask_odd, vreinterpretq_u32_f32(vi8x8ACE9BDF.val[1])));
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x8ACE, vget_high_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x8ACE, vget_high_f32(vw0123), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x8ACE, vget_high_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x8ACE, vget_low_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x8ACE, vget_low_f32(vw89AB), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x8ACE, vget_low_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x8ACE, vget_low_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x8ACE, vget_low_f32(vwCDEF), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x8ACE, vget_low_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x8ACE, vget_high_f32(vwGHIJ), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x8ACE, vget_high_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x8ACE, vget_high_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x8ACE, vget_high_f32(vwKLMN), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x8ACE, vget_high_f32(vwKLMN), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x9BDF, vget_low_f32(vw4567), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x9BDF, vget_low_f32(vw4567), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x9BDF, vget_low_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x9BDF, vget_low_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x9BDF, vget_low_f32(vw89AB), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x9BDF, vget_low_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x9BDF, vget_high_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x9BDF, vget_high_f32(vwCDEF), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x9BDF, vget_high_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x9BDF, vget_high_f32(vwGHIJ), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x9BDF, vget_high_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x9BDF, vwOP, 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x9BDF, vwOP, 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x9BDF, vwOP, 0);
+
+ const float32x4_t vi0x68AC = vextq_f32(vi0x0246, vi0x8ACE, 3);
+ const float32x4_t vi1x68AC = vextq_f32(vi1x0246, vi1x8ACE, 3);
+ const float32x4_t vi2x68AC = vextq_f32(vi2x0246, vi2x8ACE, 3);
+ const float32x4_t vi3x68AC = vextq_f32(vi3x0246, vi3x8ACE, 3);
+ const float32x4_t vi4x68AC = vextq_f32(vi4x0246, vi4x8ACE, 3);
+ const float32x4_t vi5x68AC = vextq_f32(vi5x0246, vi5x8ACE, 3);
+ const float32x4_t vi6x68AC = vextq_f32(vi6x0246, vi6x8ACE, 3);
+ const float32x4_t vi7x68AC = vextq_f32(vi7x0246, vi7x8ACE, 3);
+ const float32x4_t vi8x68AC = vextq_f32(vi8x0246, vi8x8ACE, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x68AC, vget_low_f32(vw0123), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x68AC, vget_low_f32(vw0123), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x68AC, vget_low_f32(vw0123), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x68AC, vget_high_f32(vw4567), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x68AC, vget_high_f32(vw4567), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x68AC, vget_high_f32(vw4567), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x68AC, vget_high_f32(vw89AB), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x68AC, vget_high_f32(vw89AB), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x68AC, vget_high_f32(vw89AB), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x68AC, vget_low_f32(vwGHIJ), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x68AC, vget_low_f32(vwGHIJ), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x68AC, vget_low_f32(vwGHIJ), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x68AC, vget_low_f32(vwKLMN), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x68AC, vget_low_f32(vwKLMN), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x68AC, vget_low_f32(vwKLMN), 1);
+
+ const float32x4_t vi0x79BD = vextq_f32(vi0x1357, vi0x9BDF, 3);
+ const float32x4_t vi1x79BD = vextq_f32(vi1x1357, vi1x9BDF, 3);
+ const float32x4_t vi2x79BD = vextq_f32(vi2x1357, vi2x9BDF, 3);
+ const float32x4_t vi3x79BD = vextq_f32(vi3x1357, vi3x9BDF, 3);
+ const float32x4_t vi4x79BD = vextq_f32(vi4x1357, vi4x9BDF, 3);
+ const float32x4_t vi5x79BD = vextq_f32(vi5x1357, vi5x9BDF, 3);
+ const float32x4_t vi6x79BD = vextq_f32(vi6x1357, vi6x9BDF, 3);
+ const float32x4_t vi7x79BD = vextq_f32(vi7x1357, vi7x9BDF, 3);
+ const float32x4_t vi8x79BD = vextq_f32(vi8x1357, vi8x9BDF, 3);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0x79BD, vget_high_f32(vw0123), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2x79BD, vget_high_f32(vw0123), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4x79BD, vget_high_f32(vw0123), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1x79BD, vget_high_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3x79BD, vget_high_f32(vw4567), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5x79BD, vget_high_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2x79BD, vget_low_f32(vwCDEF), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4x79BD, vget_low_f32(vwCDEF), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6x79BD, vget_low_f32(vwCDEF), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3x79BD, vget_low_f32(vwGHIJ), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5x79BD, vget_low_f32(vwGHIJ), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7x79BD, vget_low_f32(vwGHIJ), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4x79BD, vget_high_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6x79BD, vget_high_f32(vwKLMN), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8x79BD, vget_high_f32(vwKLMN), 0);
+
+ const float32x4_t vzero = vmovq_n_f32(0.0f);
+ const float32x4_t vi0xACEG = vextq_f32(vi0x8ACE, vzero, 1);
+ const float32x4_t vi1xACEG = vextq_f32(vi1x8ACE, vzero, 1);
+ const float32x4_t vi2xACEG = vextq_f32(vi2x8ACE, vzero, 1);
+ const float32x4_t vi3xACEG = vextq_f32(vi3x8ACE, vzero, 1);
+ const float32x4_t vi4xACEG = vextq_f32(vi4x8ACE, vzero, 1);
+ const float32x4_t vi5xACEG = vextq_f32(vi5x8ACE, vzero, 1);
+ const float32x4_t vi6xACEG = vextq_f32(vi6x8ACE, vzero, 1);
+ const float32x4_t vi7xACEG = vextq_f32(vi7x8ACE, vzero, 1);
+ const float32x4_t vi8xACEG = vextq_f32(vi8x8ACE, vzero, 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi0xACEG, vget_low_f32(vw4567), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi2xACEG, vget_low_f32(vw4567), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi4xACEG, vget_low_f32(vw4567), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi1xACEG, vget_high_f32(vw89AB), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi3xACEG, vget_high_f32(vw89AB), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi5xACEG, vget_high_f32(vw89AB), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi2xACEG, vget_high_f32(vwCDEF), 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi4xACEG, vget_high_f32(vwCDEF), 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi6xACEG, vget_high_f32(vwCDEF), 1);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi3xACEG, vget_low_f32(vwKLMN), 0);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi5xACEG, vget_low_f32(vwKLMN), 0);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi7xACEG, vget_low_f32(vwKLMN), 0);
+
+ vo0p0 = vfmaq_lane_f32(vo0p0, vi4xACEG, vwOP, 1);
+ vo1p0 = vfmaq_lane_f32(vo1p0, vi6xACEG, vwOP, 1);
+ vo2p0 = vfmaq_lane_f32(vo2p0, vi8xACEG, vwOP, 1);
+
+
+ float32x4_t vo0 = vmaxq_f32(vo0p0, vmin);
+ float32x4_t vo1 = vmaxq_f32(vo1p0, vmin);
+ float32x4_t vo2 = vmaxq_f32(vo2p0, vmin);
+
+ vo0 = vminq_f32(vo0, vmax);
+ vo1 = vminq_f32(vo1, vmax);
+ vo2 = vminq_f32(vo2, vmax);
+
+ size_t w_tmp = (w + 1 * sizeof(float)) / (2 * sizeof(float));
+ if XNN_LIKELY(w_tmp >= 4) {
+ vst1q_f32(o2, vo2); o2 += 4;
+ vst1q_f32(o1, vo1); o1 += 4;
+ vst1q_f32(o0, vo0); o0 += 4;
+ } else {
+ float32x2_t vo0_lo = vget_low_f32(vo0);
+ float32x2_t vo1_lo = vget_low_f32(vo1);
+ float32x2_t vo2_lo = vget_low_f32(vo2);
+ if (w_tmp & 2) {
+ vst1_f32(o2, vo2_lo); o2 += 2;
+ vst1_f32(o1, vo1_lo); o1 += 2;
+ vst1_f32(o0, vo0_lo); o0 += 2;
+
+ vo0_lo = vget_high_f32(vo0);
+ vo1_lo = vget_high_f32(vo1);
+ vo2_lo = vget_high_f32(vo2);
+ }
+ if (w_tmp & 1) {
+ vst1_lane_f32(o2, vo2_lo, 0); o2 += 1;
+ vst1_lane_f32(o1, vo1_lo, 0); o1 += 1;
+ vst1_lane_f32(o0, vo0_lo, 0); o0 += 1;
+ }
+ }
+ }
+
+ i0 = (const float*) ((uintptr_t) i6 - input_decrement);
+ i1 = (const float*) ((uintptr_t) i7 - input_decrement);
+ i2 = (const float*) ((uintptr_t) i8 - input_decrement);
+ i3 = (const float*) ((uintptr_t) i2 + input_width);
+ i4 = (const float*) ((uintptr_t) i3 + input_width);
+ i5 = (const float*) ((uintptr_t) i4 + input_width);
+ i6 = (const float*) ((uintptr_t) i5 + input_width);
+ i7 = (const float*) ((uintptr_t) i6 + input_width);
+ i8 = (const float*) ((uintptr_t) i7 + input_width);
+
+ o0 = o2;
+ o1 = (float*) ((uintptr_t) o0 + output_width);
+ o2 = (float*) ((uintptr_t) o1 + output_width);
+
+ output_height = doz(output_height, 3);
+ padded_input_height = doz(padded_input_height, 6);
+ } while (output_height != 0);
+}
diff --git a/src/xnnpack/dwconv.h b/src/xnnpack/dwconv.h
index 940f2d6..0f38f13 100644
--- a/src/xnnpack/dwconv.h
+++ b/src/xnnpack/dwconv.h
@@ -460,7 +460,26 @@
DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5p2__scalar_2x1_acc3)
DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5p2__scalar_3x1_acc2)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4)
DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc2)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3)
+DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2)
DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__psimd_1x4_acc2)
DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__scalar_1x1)
DECLARE_F32_DWCONV2D_CHW_MINMAX_UKERNEL_FUNCTION(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__scalar_2x1)
diff --git a/test/f32-dwconv2d-chw.cc b/test/f32-dwconv2d-chw.cc
index af02ed8..bf1c91b 100644
--- a/test/f32-dwconv2d-chw.cc
+++ b/test/f32-dwconv2d-chw.cc
@@ -7481,6 +7481,1936 @@
#endif // XNN_ARCH_ARM64
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, output_height_eq_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, output_height_div_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 8; input_height < 32; input_height += 4) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, output_height_lt_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, output_height_gt_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 5; input_height < 9; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 14; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, output_height_eq_3) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 5; input_height < 7; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, output_height_div_3) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 12; input_height < 48; input_height += 6) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, output_height_lt_3) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, output_height_gt_3) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 7; input_height < 13; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 20; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC2, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC2, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC2, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC2, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC2, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC2, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC2, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC3, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC3, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC3, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC3, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC3, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC3, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC3, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC4, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC4, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC4, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC4, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC4, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC4, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC4, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC5, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC5, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC5, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC5, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC5, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC5, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_1X4_ACC5, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, output_height_eq_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, output_height_div_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 8; input_height < 32; input_height += 4) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, output_height_lt_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, output_height_gt_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 5; input_height < 9; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC2, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 14; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, output_height_eq_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, output_height_div_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 8; input_height < 32; input_height += 4) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, output_height_lt_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, output_height_gt_2) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 5; input_height < 9; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_2X4_ACC3, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 14; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, output_height_eq_3) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 5; input_height < 7; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, output_height_div_3) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 12; input_height < 48; input_height += 6) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, output_height_lt_3) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 1; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, output_height_gt_3) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 7; input_height < 13; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEON_3X4_ACC2, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t input_height = 2; input_height < 20; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, output_height_eq_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, output_height_div_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 8; input_height < 32; input_height += 4) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, output_height_lt_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, output_height_gt_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 5; input_height < 9; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 14; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, output_height_eq_3) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 5; input_height < 7; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, output_height_div_3) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 12; input_height < 48; input_height += 6) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, output_height_lt_3) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, output_height_gt_3) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 7; input_height < 13; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 20; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
#if XNN_ARCH_ARM64
TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC2, output_width_eq_4) {
TEST_REQUIRES_ARM_NEON_FMA;
@@ -7609,6 +9539,888 @@
#endif // XNN_ARCH_ARM64
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC3, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC3, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC3, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC3, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC3, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC3, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC3, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC4, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC4, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC4, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC4, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC4, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC4, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC4, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC5, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC5, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC5, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC5, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(2)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC5, output_height_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC5, output_height_gt_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_1X4_ACC5, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 8; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, output_height_eq_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, output_height_div_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 8; input_height < 32; input_height += 4) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, output_height_lt_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, output_height_gt_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 5; input_height < 9; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC2, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 14; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(4)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, output_height_eq_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 3; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, output_height_div_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 8; input_height < 32; input_height += 4) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, output_height_lt_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 3; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, output_height_gt_2) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 5; input_height < 9; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_2X4_ACC3, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 14; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, output_width_eq_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 7; input_width < 9; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, output_width_div_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 16; input_width < 64; input_width += 8) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, output_width_lt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 1; input_width < 7; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(8)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, output_width_gt_4) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_width = 9; input_width < 17; input_width++) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(6)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, output_height_eq_3) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 5; input_height < 7; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, output_height_div_3) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 12; input_height < 48; input_height += 6) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, output_height_lt_3) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 1; input_height < 5; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, output_height_gt_3) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 7; input_height < 13; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(2)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+ }
+
+ TEST(F32_DWCONV2D_CHW_5X5S2P2__NEONFMA_3X4_ACC2, padding_top_eq_1) {
+ TEST_REQUIRES_ARM_NEON_FMA;
+ for (size_t input_height = 2; input_height < 20; input_height++) {
+ for (size_t input_width = 1; input_width < 41; input_width += 7) {
+ DWConv2DMicrokernelTester()
+ .input_width(input_width)
+ .input_height(input_height)
+ .kernel_height(5)
+ .kernel_width(5)
+ .subsampling(2)
+ .padding_left(2)
+ .padding_right(2)
+ .padding_top(1)
+ .padding_bottom(2)
+ .Test(xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM64
+
+
#if XNN_ARCH_X86 || XNN_ARCH_X86_64
TEST(F32_DWCONV2D_CHW_3X3P1__SSE_1X4, output_width_eq_4) {
TEST_REQUIRES_X86_SSE;
diff --git a/test/f32-dwconv2d-chw.yaml b/test/f32-dwconv2d-chw.yaml
index cfb0fd8..8a753cc 100644
--- a/test/f32-dwconv2d-chw.yaml
+++ b/test/f32-dwconv2d-chw.yaml
@@ -126,9 +126,46 @@
- name: xnn_f32_dwconv2d_chw_ukernel_5x5p2__neonfma_4x4_acc2
arch:
- aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc2
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc3
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc4
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_1x4_acc5
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc2
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_2x4_acc3
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neon_3x4_acc2
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4
+ arch:
+ - aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4
+ arch:
+ - aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4
+ arch:
+ - aarch64
- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc2
arch:
- aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc3
+ arch:
+ - aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc4
+ arch:
+ - aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_1x4_acc5
+ arch:
+ - aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc2
+ arch:
+ - aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_2x4_acc3
+ arch:
+ - aarch64
+- name: xnn_f32_dwconv2d_chw_ukernel_5x5s2p2__neonfma_3x4_acc2
+ arch:
+ - aarch64
- name: xnn_f32_dwconv2d_chw_ukernel_3x3p1__sse_1x4
- name: xnn_f32_dwconv2d_chw_ukernel_3x3p1__sse_2x4
- name: xnn_f32_dwconv2d_chw_ukernel_3x3p1__sse_3x4