AVX & AVX512F versions of binary elementwise micro-kernels
PiperOrigin-RevId: 284867789
diff --git a/src/f32-vbinary/gen/vadd-avx-x16.c b/src/f32-vbinary/gen/vadd-avx-x16.c
new file mode 100644
index 0000000..688031a
--- /dev/null
+++ b/src/f32-vbinary/gen/vadd-avx-x16.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vadd_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ const __m256 vb89ABCDEF = _mm256_loadu_ps(b + 8);
+ b += 16;
+
+ __m256 vy01234567 = _mm256_add_ps(va01234567, vb01234567);
+ __m256 vy89ABCDEF = _mm256_add_ps(va89ABCDEF, vb89ABCDEF);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_add_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_add_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vadd-avx-x8.c b/src/f32-vbinary/gen/vadd-avx-x8.c
new file mode 100644
index 0000000..14cced7
--- /dev/null
+++ b/src/f32-vbinary/gen/vadd-avx-x8.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vadd_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy01234567 = _mm256_add_ps(va01234567, vb01234567);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_add_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_add_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vadd-avx512f-x16.c b/src/f32-vbinary/gen/vadd-avx512f-x16.c
new file mode 100644
index 0000000..93a2ea2
--- /dev/null
+++ b/src/f32-vbinary/gen/vadd-avx512f-x16.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vadd_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_add_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_add_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_add_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vadd-avx512f-x32.c b/src/f32-vbinary/gen/vadd-avx512f-x32.c
new file mode 100644
index 0000000..4c40668
--- /dev/null
+++ b/src/f32-vbinary/gen/vadd-avx512f-x32.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vadd_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ const __m512 vbGHIJKLMNOPQRSTUV = _mm512_loadu_ps(b + 16);
+ b += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_add_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_add_ps(vaGHIJKLMNOPQRSTUV, vbGHIJKLMNOPQRSTUV);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_add_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_add_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vaddc-avx-x16.c b/src/f32-vbinary/gen/vaddc-avx-x16.c
new file mode 100644
index 0000000..62dbc25
--- /dev/null
+++ b/src/f32-vbinary/gen/vaddc-avx-x16.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vaddc_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ __m256 vy01234567 = _mm256_add_ps(va01234567, vb);
+ __m256 vy89ABCDEF = _mm256_add_ps(va89ABCDEF, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_add_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_add_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vaddc-avx-x8.c b/src/f32-vbinary/gen/vaddc-avx-x8.c
new file mode 100644
index 0000000..1805236
--- /dev/null
+++ b/src/f32-vbinary/gen/vaddc-avx-x8.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vaddc_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy01234567 = _mm256_add_ps(va01234567, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_add_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_add_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vaddc-avx512f-x16.c b/src/f32-vbinary/gen/vaddc-avx512f-x16.c
new file mode 100644
index 0000000..772889c
--- /dev/null
+++ b/src/f32-vbinary/gen/vaddc-avx512f-x16.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vaddc_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_add_ps(va0123456789ABCDEF, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_add_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_add_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vaddc-avx512f-x32.c b/src/f32-vbinary/gen/vaddc-avx512f-x32.c
new file mode 100644
index 0000000..d25eaed
--- /dev/null
+++ b/src/f32-vbinary/gen/vaddc-avx512f-x32.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vaddc_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_add_ps(va0123456789ABCDEF, vb);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_add_ps(vaGHIJKLMNOPQRSTUV, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_add_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_add_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vdiv-avx-x16.c b/src/f32-vbinary/gen/vdiv-avx-x16.c
new file mode 100644
index 0000000..2e95dff
--- /dev/null
+++ b/src/f32-vbinary/gen/vdiv-avx-x16.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vdiv_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ const __m256 vb89ABCDEF = _mm256_loadu_ps(b + 8);
+ b += 16;
+
+ __m256 vy01234567 = _mm256_div_ps(va01234567, vb01234567);
+ __m256 vy89ABCDEF = _mm256_div_ps(va89ABCDEF, vb89ABCDEF);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_div_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_div_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vdiv-avx-x8.c b/src/f32-vbinary/gen/vdiv-avx-x8.c
new file mode 100644
index 0000000..7e7e8c5
--- /dev/null
+++ b/src/f32-vbinary/gen/vdiv-avx-x8.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vdiv_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy01234567 = _mm256_div_ps(va01234567, vb01234567);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_div_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_div_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vdiv-avx512f-x16.c b/src/f32-vbinary/gen/vdiv-avx512f-x16.c
new file mode 100644
index 0000000..aa1e141
--- /dev/null
+++ b/src/f32-vbinary/gen/vdiv-avx512f-x16.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vdiv_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_div_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_div_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_div_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vdiv-avx512f-x32.c b/src/f32-vbinary/gen/vdiv-avx512f-x32.c
new file mode 100644
index 0000000..e335a2d
--- /dev/null
+++ b/src/f32-vbinary/gen/vdiv-avx512f-x32.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vdiv_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ const __m512 vbGHIJKLMNOPQRSTUV = _mm512_loadu_ps(b + 16);
+ b += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_div_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_div_ps(vaGHIJKLMNOPQRSTUV, vbGHIJKLMNOPQRSTUV);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_div_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_div_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vdivc-avx-x16.c b/src/f32-vbinary/gen/vdivc-avx-x16.c
new file mode 100644
index 0000000..d9ed7bf
--- /dev/null
+++ b/src/f32-vbinary/gen/vdivc-avx-x16.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vdivc_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ __m256 vy01234567 = _mm256_div_ps(va01234567, vb);
+ __m256 vy89ABCDEF = _mm256_div_ps(va89ABCDEF, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_div_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_div_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vdivc-avx-x8.c b/src/f32-vbinary/gen/vdivc-avx-x8.c
new file mode 100644
index 0000000..5eb7b90
--- /dev/null
+++ b/src/f32-vbinary/gen/vdivc-avx-x8.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vdivc_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy01234567 = _mm256_div_ps(va01234567, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_div_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_div_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vdivc-avx512f-x16.c b/src/f32-vbinary/gen/vdivc-avx512f-x16.c
new file mode 100644
index 0000000..9389376
--- /dev/null
+++ b/src/f32-vbinary/gen/vdivc-avx512f-x16.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vdivc_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_div_ps(va0123456789ABCDEF, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_div_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_div_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vdivc-avx512f-x32.c b/src/f32-vbinary/gen/vdivc-avx512f-x32.c
new file mode 100644
index 0000000..363cfee
--- /dev/null
+++ b/src/f32-vbinary/gen/vdivc-avx512f-x32.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vdivc_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_div_ps(va0123456789ABCDEF, vb);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_div_ps(vaGHIJKLMNOPQRSTUV, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_div_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_div_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmax-avx-x16.c b/src/f32-vbinary/gen/vmax-avx-x16.c
new file mode 100644
index 0000000..5fa0838
--- /dev/null
+++ b/src/f32-vbinary/gen/vmax-avx-x16.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmax_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ const __m256 vb89ABCDEF = _mm256_loadu_ps(b + 8);
+ b += 16;
+
+ __m256 vy01234567 = _mm256_max_ps(va01234567, vb01234567);
+ __m256 vy89ABCDEF = _mm256_max_ps(va89ABCDEF, vb89ABCDEF);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_max_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_max_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmax-avx-x8.c b/src/f32-vbinary/gen/vmax-avx-x8.c
new file mode 100644
index 0000000..797ca42
--- /dev/null
+++ b/src/f32-vbinary/gen/vmax-avx-x8.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmax_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy01234567 = _mm256_max_ps(va01234567, vb01234567);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_max_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_max_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmax-avx512f-x16.c b/src/f32-vbinary/gen/vmax-avx512f-x16.c
new file mode 100644
index 0000000..6d58113
--- /dev/null
+++ b/src/f32-vbinary/gen/vmax-avx512f-x16.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmax_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_max_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_max_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_max_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmax-avx512f-x32.c b/src/f32-vbinary/gen/vmax-avx512f-x32.c
new file mode 100644
index 0000000..88f21e1
--- /dev/null
+++ b/src/f32-vbinary/gen/vmax-avx512f-x32.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmax_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ const __m512 vbGHIJKLMNOPQRSTUV = _mm512_loadu_ps(b + 16);
+ b += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_max_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vaGHIJKLMNOPQRSTUV, vbGHIJKLMNOPQRSTUV);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_max_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_max_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmaxc-avx-x16.c b/src/f32-vbinary/gen/vmaxc-avx-x16.c
new file mode 100644
index 0000000..fb3ab79
--- /dev/null
+++ b/src/f32-vbinary/gen/vmaxc-avx-x16.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmaxc_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ __m256 vy01234567 = _mm256_max_ps(va01234567, vb);
+ __m256 vy89ABCDEF = _mm256_max_ps(va89ABCDEF, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_max_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_max_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmaxc-avx-x8.c b/src/f32-vbinary/gen/vmaxc-avx-x8.c
new file mode 100644
index 0000000..8ae5005
--- /dev/null
+++ b/src/f32-vbinary/gen/vmaxc-avx-x8.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmaxc_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy01234567 = _mm256_max_ps(va01234567, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_max_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_max_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmaxc-avx512f-x16.c b/src/f32-vbinary/gen/vmaxc-avx512f-x16.c
new file mode 100644
index 0000000..2c0bb1e
--- /dev/null
+++ b/src/f32-vbinary/gen/vmaxc-avx512f-x16.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmaxc_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_max_ps(va0123456789ABCDEF, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_max_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_max_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmaxc-avx512f-x32.c b/src/f32-vbinary/gen/vmaxc-avx512f-x32.c
new file mode 100644
index 0000000..2353b14
--- /dev/null
+++ b/src/f32-vbinary/gen/vmaxc-avx512f-x32.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmaxc_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_max_ps(va0123456789ABCDEF, vb);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vaGHIJKLMNOPQRSTUV, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_max_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_max_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmin-avx-x16.c b/src/f32-vbinary/gen/vmin-avx-x16.c
new file mode 100644
index 0000000..c6b0384
--- /dev/null
+++ b/src/f32-vbinary/gen/vmin-avx-x16.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmin_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ const __m256 vb89ABCDEF = _mm256_loadu_ps(b + 8);
+ b += 16;
+
+ __m256 vy01234567 = _mm256_min_ps(va01234567, vb01234567);
+ __m256 vy89ABCDEF = _mm256_min_ps(va89ABCDEF, vb89ABCDEF);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_min_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_min_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmin-avx-x8.c b/src/f32-vbinary/gen/vmin-avx-x8.c
new file mode 100644
index 0000000..3e22da8
--- /dev/null
+++ b/src/f32-vbinary/gen/vmin-avx-x8.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmin_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy01234567 = _mm256_min_ps(va01234567, vb01234567);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_min_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_min_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmin-avx512f-x16.c b/src/f32-vbinary/gen/vmin-avx512f-x16.c
new file mode 100644
index 0000000..81cd33b
--- /dev/null
+++ b/src/f32-vbinary/gen/vmin-avx512f-x16.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmin_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_min_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_min_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_min_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmin-avx512f-x32.c b/src/f32-vbinary/gen/vmin-avx512f-x32.c
new file mode 100644
index 0000000..2c65a2d
--- /dev/null
+++ b/src/f32-vbinary/gen/vmin-avx512f-x32.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmin_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ const __m512 vbGHIJKLMNOPQRSTUV = _mm512_loadu_ps(b + 16);
+ b += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_min_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vaGHIJKLMNOPQRSTUV, vbGHIJKLMNOPQRSTUV);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_min_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_min_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vminc-avx-x16.c b/src/f32-vbinary/gen/vminc-avx-x16.c
new file mode 100644
index 0000000..5085cbc
--- /dev/null
+++ b/src/f32-vbinary/gen/vminc-avx-x16.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vminc_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ __m256 vy01234567 = _mm256_min_ps(va01234567, vb);
+ __m256 vy89ABCDEF = _mm256_min_ps(va89ABCDEF, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_min_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_min_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vminc-avx-x8.c b/src/f32-vbinary/gen/vminc-avx-x8.c
new file mode 100644
index 0000000..c4a4651
--- /dev/null
+++ b/src/f32-vbinary/gen/vminc-avx-x8.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vminc_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy01234567 = _mm256_min_ps(va01234567, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_min_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_min_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vminc-avx512f-x16.c b/src/f32-vbinary/gen/vminc-avx512f-x16.c
new file mode 100644
index 0000000..ddc667d
--- /dev/null
+++ b/src/f32-vbinary/gen/vminc-avx512f-x16.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vminc_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_min_ps(va0123456789ABCDEF, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_min_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_min_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vminc-avx512f-x32.c b/src/f32-vbinary/gen/vminc-avx512f-x32.c
new file mode 100644
index 0000000..9059a84
--- /dev/null
+++ b/src/f32-vbinary/gen/vminc-avx512f-x32.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vminc_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_min_ps(va0123456789ABCDEF, vb);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vaGHIJKLMNOPQRSTUV, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_min_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_min_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmul-avx-x16.c b/src/f32-vbinary/gen/vmul-avx-x16.c
new file mode 100644
index 0000000..12dc8dc
--- /dev/null
+++ b/src/f32-vbinary/gen/vmul-avx-x16.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmul_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ const __m256 vb89ABCDEF = _mm256_loadu_ps(b + 8);
+ b += 16;
+
+ __m256 vy01234567 = _mm256_mul_ps(va01234567, vb01234567);
+ __m256 vy89ABCDEF = _mm256_mul_ps(va89ABCDEF, vb89ABCDEF);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_mul_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_mul_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmul-avx-x8.c b/src/f32-vbinary/gen/vmul-avx-x8.c
new file mode 100644
index 0000000..2b2de60
--- /dev/null
+++ b/src/f32-vbinary/gen/vmul-avx-x8.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmul_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy01234567 = _mm256_mul_ps(va01234567, vb01234567);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_mul_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_mul_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmul-avx512f-x16.c b/src/f32-vbinary/gen/vmul-avx512f-x16.c
new file mode 100644
index 0000000..a85665e
--- /dev/null
+++ b/src/f32-vbinary/gen/vmul-avx512f-x16.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmul_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_mul_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_mul_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_mul_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmul-avx512f-x32.c b/src/f32-vbinary/gen/vmul-avx512f-x32.c
new file mode 100644
index 0000000..443987f
--- /dev/null
+++ b/src/f32-vbinary/gen/vmul-avx512f-x32.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmul_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ const __m512 vbGHIJKLMNOPQRSTUV = _mm512_loadu_ps(b + 16);
+ b += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_mul_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_mul_ps(vaGHIJKLMNOPQRSTUV, vbGHIJKLMNOPQRSTUV);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_mul_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_mul_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmulc-avx-x16.c b/src/f32-vbinary/gen/vmulc-avx-x16.c
new file mode 100644
index 0000000..bd73937
--- /dev/null
+++ b/src/f32-vbinary/gen/vmulc-avx-x16.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmulc_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ __m256 vy01234567 = _mm256_mul_ps(va01234567, vb);
+ __m256 vy89ABCDEF = _mm256_mul_ps(va89ABCDEF, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_mul_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_mul_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmulc-avx-x8.c b/src/f32-vbinary/gen/vmulc-avx-x8.c
new file mode 100644
index 0000000..b76e079
--- /dev/null
+++ b/src/f32-vbinary/gen/vmulc-avx-x8.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vmulc_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy01234567 = _mm256_mul_ps(va01234567, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_mul_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_mul_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmulc-avx512f-x16.c b/src/f32-vbinary/gen/vmulc-avx512f-x16.c
new file mode 100644
index 0000000..1a86801
--- /dev/null
+++ b/src/f32-vbinary/gen/vmulc-avx512f-x16.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmulc_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_mul_ps(va0123456789ABCDEF, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_mul_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_mul_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vmulc-avx512f-x32.c b/src/f32-vbinary/gen/vmulc-avx512f-x32.c
new file mode 100644
index 0000000..006bf6c
--- /dev/null
+++ b/src/f32-vbinary/gen/vmulc-avx512f-x32.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vmulc_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_mul_ps(va0123456789ABCDEF, vb);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_mul_ps(vaGHIJKLMNOPQRSTUV, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_mul_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_mul_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vrdivc-avx-x16.c b/src/f32-vbinary/gen/vrdivc-avx-x16.c
new file mode 100644
index 0000000..0efa803
--- /dev/null
+++ b/src/f32-vbinary/gen/vrdivc-avx-x16.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vrdivc_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ __m256 vy01234567 = _mm256_div_ps(vb, va01234567);
+ __m256 vy89ABCDEF = _mm256_div_ps(vb, va89ABCDEF);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_div_ps(vb, va);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_div_ps(vb, va);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vrdivc-avx-x8.c b/src/f32-vbinary/gen/vrdivc-avx-x8.c
new file mode 100644
index 0000000..80a483a
--- /dev/null
+++ b/src/f32-vbinary/gen/vrdivc-avx-x8.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vrdivc_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy01234567 = _mm256_div_ps(vb, va01234567);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_div_ps(vb, va);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_div_ps(vb, va);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vrdivc-avx512f-x16.c b/src/f32-vbinary/gen/vrdivc-avx512f-x16.c
new file mode 100644
index 0000000..f47a5d2
--- /dev/null
+++ b/src/f32-vbinary/gen/vrdivc-avx512f-x16.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vrdivc_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_div_ps(vb, va0123456789ABCDEF);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_div_ps(vb, va);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_div_ps(vb, va);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vrdivc-avx512f-x32.c b/src/f32-vbinary/gen/vrdivc-avx512f-x32.c
new file mode 100644
index 0000000..4be883d
--- /dev/null
+++ b/src/f32-vbinary/gen/vrdivc-avx512f-x32.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vrdivc_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_div_ps(vb, va0123456789ABCDEF);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_div_ps(vb, vaGHIJKLMNOPQRSTUV);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_div_ps(vb, va);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_div_ps(vb, va);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vrsubc-avx-x16.c b/src/f32-vbinary/gen/vrsubc-avx-x16.c
new file mode 100644
index 0000000..7f502a9
--- /dev/null
+++ b/src/f32-vbinary/gen/vrsubc-avx-x16.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vrsubc_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ __m256 vy01234567 = _mm256_sub_ps(vb, va01234567);
+ __m256 vy89ABCDEF = _mm256_sub_ps(vb, va89ABCDEF);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_sub_ps(vb, va);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_sub_ps(vb, va);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vrsubc-avx-x8.c b/src/f32-vbinary/gen/vrsubc-avx-x8.c
new file mode 100644
index 0000000..0c866bf
--- /dev/null
+++ b/src/f32-vbinary/gen/vrsubc-avx-x8.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vrsubc_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy01234567 = _mm256_sub_ps(vb, va01234567);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_sub_ps(vb, va);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_sub_ps(vb, va);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vrsubc-avx512f-x16.c b/src/f32-vbinary/gen/vrsubc-avx512f-x16.c
new file mode 100644
index 0000000..c54a04a
--- /dev/null
+++ b/src/f32-vbinary/gen/vrsubc-avx512f-x16.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vrsubc_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_sub_ps(vb, va0123456789ABCDEF);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_sub_ps(vb, va);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_sub_ps(vb, va);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vrsubc-avx512f-x32.c b/src/f32-vbinary/gen/vrsubc-avx512f-x32.c
new file mode 100644
index 0000000..6be807d
--- /dev/null
+++ b/src/f32-vbinary/gen/vrsubc-avx512f-x32.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vrsubc_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_sub_ps(vb, va0123456789ABCDEF);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_sub_ps(vb, vaGHIJKLMNOPQRSTUV);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_sub_ps(vb, va);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_sub_ps(vb, va);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vsub-avx-x16.c b/src/f32-vbinary/gen/vsub-avx-x16.c
new file mode 100644
index 0000000..d7bc193
--- /dev/null
+++ b/src/f32-vbinary/gen/vsub-avx-x16.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vsub_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ const __m256 vb89ABCDEF = _mm256_loadu_ps(b + 8);
+ b += 16;
+
+ __m256 vy01234567 = _mm256_sub_ps(va01234567, vb01234567);
+ __m256 vy89ABCDEF = _mm256_sub_ps(va89ABCDEF, vb89ABCDEF);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_sub_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_sub_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vsub-avx-x8.c b/src/f32-vbinary/gen/vsub-avx-x8.c
new file mode 100644
index 0000000..351dd2d
--- /dev/null
+++ b/src/f32-vbinary/gen/vsub-avx-x8.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vsub_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb01234567 = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy01234567 = _mm256_sub_ps(va01234567, vb01234567);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = _mm256_sub_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = _mm256_sub_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vsub-avx512f-x16.c b/src/f32-vbinary/gen/vsub-avx512f-x16.c
new file mode 100644
index 0000000..123ad62
--- /dev/null
+++ b/src/f32-vbinary/gen/vsub-avx512f-x16.c
@@ -0,0 +1,75 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vsub_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_sub_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_sub_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_sub_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vsub-avx512f-x32.c b/src/f32-vbinary/gen/vsub-avx512f-x32.c
new file mode 100644
index 0000000..989ec76
--- /dev/null
+++ b/src/f32-vbinary/gen/vsub-avx512f-x32.c
@@ -0,0 +1,81 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vop-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vsub_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ const __m512 vb0123456789ABCDEF = _mm512_loadu_ps(b);
+ const __m512 vbGHIJKLMNOPQRSTUV = _mm512_loadu_ps(b + 16);
+ b += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_sub_ps(va0123456789ABCDEF, vb0123456789ABCDEF);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_sub_ps(vaGHIJKLMNOPQRSTUV, vbGHIJKLMNOPQRSTUV);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = _mm512_sub_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = _mm512_sub_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vsubc-avx-x16.c b/src/f32-vbinary/gen/vsubc-avx-x16.c
new file mode 100644
index 0000000..dde9978
--- /dev/null
+++ b/src/f32-vbinary/gen/vsubc-avx-x16.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vsubc_ukernel__avx_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ const __m256 va89ABCDEF = _mm256_loadu_ps(a + 8);
+ a += 16;
+
+ __m256 vy01234567 = _mm256_sub_ps(va01234567, vb);
+ __m256 vy89ABCDEF = _mm256_sub_ps(va89ABCDEF, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+ vy89ABCDEF = _mm256_max_ps(vy89ABCDEF, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+ vy89ABCDEF = _mm256_min_ps(vy89ABCDEF, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ _mm256_storeu_ps(y + 8, vy89ABCDEF);
+ y += 16;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_sub_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_sub_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vsubc-avx-x8.c b/src/f32-vbinary/gen/vsubc-avx-x8.c
new file mode 100644
index 0000000..e2ffc06
--- /dev/null
+++ b/src/f32-vbinary/gen/vsubc-avx-x8.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+void xnn_f32_vsubc_ukernel__avx_x8(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va01234567 = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy01234567 = _mm256_sub_ps(va01234567, vb);
+
+ vy01234567 = _mm256_max_ps(vy01234567, vy_min);
+
+ vy01234567 = _mm256_min_ps(vy01234567, vy_max);
+
+ _mm256_storeu_ps(y, vy01234567);
+ y += 8;
+ }
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = _mm256_sub_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = _mm256_sub_ps(va, vb);
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vsubc-avx512f-x16.c b/src/f32-vbinary/gen/vsubc-avx512f-x16.c
new file mode 100644
index 0000000..87ea9d1
--- /dev/null
+++ b/src/f32-vbinary/gen/vsubc-avx512f-x16.c
@@ -0,0 +1,69 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vsubc_ukernel__avx512f_x16(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy0123456789ABCDEF = _mm512_sub_ps(va0123456789ABCDEF, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ y += 16;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_sub_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_sub_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/gen/vsubc-avx512f-x32.c b/src/f32-vbinary/gen/vsubc-avx512f-x32.c
new file mode 100644
index 0000000..0766745
--- /dev/null
+++ b/src/f32-vbinary/gen/vsubc-avx512f-x32.c
@@ -0,0 +1,74 @@
+// Auto-generated file. Do not edit!
+// Template: src/f32-vbinary/vopc-avx512f.c.in
+// Generator: tools/xngen
+//
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+void xnn_f32_vsubc_ukernel__avx512f_x32(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= 32 * sizeof(float); n -= 32 * sizeof(float)) {
+ const __m512 va0123456789ABCDEF = _mm512_loadu_ps(a);
+ const __m512 vaGHIJKLMNOPQRSTUV = _mm512_loadu_ps(a + 16);
+ a += 32;
+
+ __m512 vy0123456789ABCDEF = _mm512_sub_ps(va0123456789ABCDEF, vb);
+ __m512 vyGHIJKLMNOPQRSTUV = _mm512_sub_ps(vaGHIJKLMNOPQRSTUV, vb);
+
+ vy0123456789ABCDEF = _mm512_max_ps(vy0123456789ABCDEF, vy_min);
+ vyGHIJKLMNOPQRSTUV = _mm512_max_ps(vyGHIJKLMNOPQRSTUV, vy_min);
+
+ vy0123456789ABCDEF = _mm512_min_ps(vy0123456789ABCDEF, vy_max);
+ vyGHIJKLMNOPQRSTUV = _mm512_min_ps(vyGHIJKLMNOPQRSTUV, vy_max);
+
+ _mm512_storeu_ps(y, vy0123456789ABCDEF);
+ _mm512_storeu_ps(y + 16, vyGHIJKLMNOPQRSTUV);
+ y += 32;
+ }
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = _mm512_sub_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = _mm512_sub_ps(va, vb);
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/vop-avx.c.in b/src/f32-vbinary/vop-avx.c.in
new file mode 100644
index 0000000..29a4a85
--- /dev/null
+++ b/src/f32-vbinary/vop-avx.c.in
@@ -0,0 +1,93 @@
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+$assert BATCH_TILE % 8 == 0
+$assert BATCH_TILE >= 8
+$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+$assert OP in ["ADD", "DIV", "MAX", "MIN", "MUL", "SUB"]
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+$_MM256_OP_PS = {
+$ "ADD": lambda x, y: "_mm256_add_ps(%s, %s)" % (x, y),
+$ "DIV": lambda x, y: "_mm256_div_ps(%s, %s)" % (x, y),
+$ "MAX": lambda x, y: "_mm256_max_ps(%s, %s)" % (x, y),
+$ "MIN": lambda x, y: "_mm256_min_ps(%s, %s)" % (x, y),
+$ "MUL": lambda x, y: "_mm256_mul_ps(%s, %s)" % (x, y),
+$ "SUB": lambda x, y: "_mm256_sub_ps(%s, %s)" % (x, y),
+$}[OP]
+void xnn_f32_v${OP.lower()}_ukernel__avx_x${BATCH_TILE}(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) {
+ const __m256 va${ABC[0:8]} = _mm256_loadu_ps(a);
+ $for N in range(8, BATCH_TILE, 8):
+ const __m256 va${ABC[N:N+8]} = _mm256_loadu_ps(a + ${N});
+ a += ${BATCH_TILE};
+
+ const __m256 vb${ABC[0:8]} = _mm256_loadu_ps(b);
+ $for N in range(8, BATCH_TILE, 8):
+ const __m256 vb${ABC[N:N+8]} = _mm256_loadu_ps(b + ${N});
+ b += ${BATCH_TILE};
+
+ $for N in range(0, BATCH_TILE, 8):
+ __m256 vy${ABC[N:N+8]} = ${_MM256_OP_PS("va" + ABC[N:N+8], "vb" + ABC[N:N+8])};
+
+ $for N in range(0, BATCH_TILE, 8):
+ vy${ABC[N:N+8]} = _mm256_max_ps(vy${ABC[N:N+8]}, vy_min);
+
+ $for N in range(0, BATCH_TILE, 8):
+ vy${ABC[N:N+8]} = _mm256_min_ps(vy${ABC[N:N+8]}, vy_max);
+
+ _mm256_storeu_ps(y, vy${ABC[0:8]});
+ $for N in range(8, BATCH_TILE, 8):
+ _mm256_storeu_ps(y + ${N}, vy${ABC[N:N+8]});
+ y += ${BATCH_TILE};
+ }
+ $if BATCH_TILE >= 8:
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ const __m256 vb = _mm256_loadu_ps(b);
+ b += 8;
+
+ __m256 vy = ${_MM256_OP_PS("va", "vb")};
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+ const __m256 vb = _mm256_maskload_ps(b, vmask);
+
+ __m256 vy = ${_MM256_OP_PS("va", "vb")};
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/vop-avx512f.c.in b/src/f32-vbinary/vop-avx512f.c.in
new file mode 100644
index 0000000..84594a3
--- /dev/null
+++ b/src/f32-vbinary/vop-avx512f.c.in
@@ -0,0 +1,93 @@
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+$assert BATCH_TILE % 16 == 0
+$assert BATCH_TILE >= 16
+$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+$assert OP in ["ADD", "DIV", "MAX", "MIN", "MUL", "SUB"]
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+$_MM512_OP_PS = {
+$ "ADD": lambda x, y: "_mm512_add_ps(%s, %s)" % (x, y),
+$ "DIV": lambda x, y: "_mm512_div_ps(%s, %s)" % (x, y),
+$ "MAX": lambda x, y: "_mm512_max_ps(%s, %s)" % (x, y),
+$ "MIN": lambda x, y: "_mm512_min_ps(%s, %s)" % (x, y),
+$ "MUL": lambda x, y: "_mm512_mul_ps(%s, %s)" % (x, y),
+$ "SUB": lambda x, y: "_mm512_sub_ps(%s, %s)" % (x, y),
+$}[OP]
+void xnn_f32_v${OP.lower()}_ukernel__avx512f_x${BATCH_TILE}(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) {
+ const __m512 va${ABC[0:16]} = _mm512_loadu_ps(a);
+ $for N in range(16, BATCH_TILE, 16):
+ const __m512 va${ABC[N:N+16]} = _mm512_loadu_ps(a + ${N});
+ a += ${BATCH_TILE};
+
+ const __m512 vb${ABC[0:16]} = _mm512_loadu_ps(b);
+ $for N in range(16, BATCH_TILE, 16):
+ const __m512 vb${ABC[N:N+16]} = _mm512_loadu_ps(b + ${N});
+ b += ${BATCH_TILE};
+
+ $for N in range(0, BATCH_TILE, 16):
+ __m512 vy${ABC[N:N+16]} = ${_MM512_OP_PS("va" + ABC[N:N+16], "vb" + ABC[N:N+16])};
+
+ $for N in range(0, BATCH_TILE, 16):
+ vy${ABC[N:N+16]} = _mm512_max_ps(vy${ABC[N:N+16]}, vy_min);
+
+ $for N in range(0, BATCH_TILE, 16):
+ vy${ABC[N:N+16]} = _mm512_min_ps(vy${ABC[N:N+16]}, vy_max);
+
+ _mm512_storeu_ps(y, vy${ABC[0:16]});
+ $for N in range(16, BATCH_TILE, 16):
+ _mm512_storeu_ps(y + ${N}, vy${ABC[N:N+16]});
+ y += ${BATCH_TILE};
+ }
+ $if BATCH_TILE >= 16:
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ const __m512 vb = _mm512_loadu_ps(b);
+ b += 16;
+
+ __m512 vy = ${_MM512_OP_PS("va", "vb")};
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+ const __m512 vb = _mm512_maskz_loadu_ps(vmask, b);
+
+ __m512 vy = ${_MM512_OP_PS("va", "vb")};
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/vopc-avx.c.in b/src/f32-vbinary/vopc-avx.c.in
new file mode 100644
index 0000000..c516482
--- /dev/null
+++ b/src/f32-vbinary/vopc-avx.c.in
@@ -0,0 +1,87 @@
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+$assert BATCH_TILE % 8 == 0
+$assert BATCH_TILE >= 8
+$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+$assert OP in ["ADD", "DIV", "RDIV", "MAX", "MIN", "MUL", "SUB", "RSUB"]
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+static const int32_t mask_table[14] = {-1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0};
+
+$_MM256_OP_PS = {
+$ "ADD": lambda x: "_mm256_add_ps(%s, vb)" % x,
+$ "DIV": lambda x: "_mm256_div_ps(%s, vb)" % x,
+$ "RDIV": lambda x: "_mm256_div_ps(vb, %s)" % x,
+$ "MAX": lambda x: "_mm256_max_ps(%s, vb)" % x,
+$ "MIN": lambda x: "_mm256_min_ps(%s, vb)" % x,
+$ "MUL": lambda x: "_mm256_mul_ps(%s, vb)" % x,
+$ "SUB": lambda x: "_mm256_sub_ps(%s, vb)" % x,
+$ "RSUB": lambda x: "_mm256_sub_ps(vb, %s)" % x,
+$}[OP]
+void xnn_f32_v${OP.lower()}c_ukernel__avx_x${BATCH_TILE}(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m256 vy_min = _mm256_broadcast_ps((const __m128*) params->sse.min);
+ const __m256 vy_max = _mm256_broadcast_ps((const __m128*) params->sse.max);
+
+ const __m256 vb = _mm256_broadcast_ss(b);
+ for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) {
+ const __m256 va${ABC[0:8]} = _mm256_loadu_ps(a);
+ $for N in range(8, BATCH_TILE, 8):
+ const __m256 va${ABC[N:N+8]} = _mm256_loadu_ps(a + ${N});
+ a += ${BATCH_TILE};
+
+ $for N in range(0, BATCH_TILE, 8):
+ __m256 vy${ABC[N:N+8]} = ${_MM256_OP_PS("va" + ABC[N:N+8])};
+
+ $for N in range(0, BATCH_TILE, 8):
+ vy${ABC[N:N+8]} = _mm256_max_ps(vy${ABC[N:N+8]}, vy_min);
+
+ $for N in range(0, BATCH_TILE, 8):
+ vy${ABC[N:N+8]} = _mm256_min_ps(vy${ABC[N:N+8]}, vy_max);
+
+ _mm256_storeu_ps(y, vy${ABC[0:8]});
+ $for N in range(8, BATCH_TILE, 8):
+ _mm256_storeu_ps(y + ${N}, vy${ABC[N:N+8]});
+ y += ${BATCH_TILE};
+ }
+ $if BATCH_TILE >= 8:
+ for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
+ const __m256 va = _mm256_loadu_ps(a);
+ a += 8;
+
+ __m256 vy = ${_MM256_OP_PS("va")};
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_storeu_ps(y, vy);
+ y += 8;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 7 * sizeof(float));
+ __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) &mask_table[7] - n));
+
+ const __m256 va = _mm256_maskload_ps(a, vmask);
+
+ __m256 vy = ${_MM256_OP_PS("va")};
+ vy = _mm256_max_ps(vy, vy_min);
+ vy = _mm256_min_ps(vy, vy_max);
+ _mm256_maskstore_ps(y, vmask, vy);
+ }
+}
diff --git a/src/f32-vbinary/vopc-avx512f.c.in b/src/f32-vbinary/vopc-avx512f.c.in
new file mode 100644
index 0000000..72076e8
--- /dev/null
+++ b/src/f32-vbinary/vopc-avx512f.c.in
@@ -0,0 +1,87 @@
+// Copyright 2019 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+$assert BATCH_TILE % 16 == 0
+$assert BATCH_TILE >= 16
+$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+$assert OP in ["ADD", "DIV", "RDIV", "MAX", "MIN", "MUL", "SUB", "RSUB"]
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/common.h>
+#include <xnnpack/vbinary.h>
+
+
+$_MM512_OP_PS = {
+$ "ADD": lambda x: "_mm512_add_ps(%s, vb)" % x,
+$ "DIV": lambda x: "_mm512_div_ps(%s, vb)" % x,
+$ "RDIV": lambda x: "_mm512_div_ps(vb, %s)" % x,
+$ "MAX": lambda x: "_mm512_max_ps(%s, vb)" % x,
+$ "MIN": lambda x: "_mm512_min_ps(%s, vb)" % x,
+$ "MUL": lambda x: "_mm512_mul_ps(%s, vb)" % x,
+$ "SUB": lambda x: "_mm512_sub_ps(%s, vb)" % x,
+$ "RSUB": lambda x: "_mm512_sub_ps(vb, %s)" % x,
+$}[OP]
+void xnn_f32_v${OP.lower()}c_ukernel__avx512f_x${BATCH_TILE}(
+ size_t n,
+ const float* a,
+ const float* b,
+ float* y,
+ const union xnn_f32_output_params params[restrict static 1])
+{
+ assert(n != 0);
+ assert(n % sizeof(float) == 0);
+
+ const __m512 vy_min = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.min));
+ const __m512 vy_max = _mm512_broadcast_f32x4(_mm_load_ps(params->sse.max));
+
+ const __m512 vb = _mm512_set1_ps(*b);
+ for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) {
+ const __m512 va${ABC[0:16]} = _mm512_loadu_ps(a);
+ $for N in range(16, BATCH_TILE, 16):
+ const __m512 va${ABC[N:N+16]} = _mm512_loadu_ps(a + ${N});
+ a += ${BATCH_TILE};
+
+ $for N in range(0, BATCH_TILE, 16):
+ __m512 vy${ABC[N:N+16]} = ${_MM512_OP_PS("va" + ABC[N:N+16])};
+
+ $for N in range(0, BATCH_TILE, 16):
+ vy${ABC[N:N+16]} = _mm512_max_ps(vy${ABC[N:N+16]}, vy_min);
+
+ $for N in range(0, BATCH_TILE, 16):
+ vy${ABC[N:N+16]} = _mm512_min_ps(vy${ABC[N:N+16]}, vy_max);
+
+ _mm512_storeu_ps(y, vy${ABC[0:16]});
+ $for N in range(16, BATCH_TILE, 16):
+ _mm512_storeu_ps(y + ${N}, vy${ABC[N:N+16]});
+ y += ${BATCH_TILE};
+ }
+ $if BATCH_TILE >= 16:
+ for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
+ const __m512 va = _mm512_loadu_ps(a);
+ a += 16;
+
+ __m512 vy = ${_MM512_OP_PS("va")};
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_storeu_ps(y, vy);
+ y += 16;
+ }
+ if XNN_UNLIKELY(n != 0) {
+ assert(n >= 1 * sizeof(float));
+ assert(n <= 15 * sizeof(float));
+ // Prepare mask for valid 32-bit elements (depends on n).
+ n >>= 2 /* log2(sizeof(float)) */;
+ const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
+
+ const __m512 va = _mm512_maskz_loadu_ps(vmask, a);
+
+ __m512 vy = ${_MM512_OP_PS("va")};
+ vy = _mm512_max_ps(vy, vy_min);
+ vy = _mm512_min_ps(vy, vy_max);
+ _mm512_mask_storeu_ps(y, vmask, vy);
+ }
+}