Quantized GEMM/IGEMM microkernels bump kc to be a multiple of channels.
Rewind A pointers by KC.
Remove last partial channel of remainder code. Its now handled by main loop.
PiperOrigin-RevId: 360231001
diff --git a/src/qs8-igemm/MRx16c8-avx512skx.c.in b/src/qs8-igemm/MRx16c8-avx512skx.c.in
index 0244207..af17147 100644
--- a/src/qs8-igemm/MRx16c8-avx512skx.c.in
+++ b/src/qs8-igemm/MRx16c8-avx512skx.c.in
@@ -12,6 +12,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
@@ -38,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
diff --git a/src/qs8-igemm/MRx4c2-sse.c.in b/src/qs8-igemm/MRx4c2-sse.c.in
index b917f66..d111fa1 100644
--- a/src/qs8-igemm/MRx4c2-sse.c.in
+++ b/src/qs8-igemm/MRx4c2-sse.c.in
@@ -18,6 +18,7 @@
#include <${SSE_HEADER}>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
$ISA = {2: "sse2", 3: "ssse3", 4: "sse41", 5: "xop"}[SSE]
@@ -46,6 +47,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
@@ -180,20 +182,6 @@
$else:
vacc${M}x0123 = _mm_add_epi32(vacc${M}x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa${M}, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- $for M in range(MR):
- $if SSE == 5:
- vacc${M}x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa${M}, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc${M}x0123);
- $else:
- vacc${M}x0123 = _mm_add_epi32(vacc${M}x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa${M}, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/MRx4c8-sse.c.in b/src/qs8-igemm/MRx4c8-sse.c.in
index 6a9b5df..030d144 100644
--- a/src/qs8-igemm/MRx4c8-sse.c.in
+++ b/src/qs8-igemm/MRx4c8-sse.c.in
@@ -18,6 +18,7 @@
#include <${SSE_HEADER}>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
$ISA = {2: "sse2", 3: "ssse3", 4: "sse41", 5: "xop"}[SSE]
@@ -46,6 +47,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
diff --git a/src/qs8-igemm/MRx4c8-wasmsimd.c.in b/src/qs8-igemm/MRx4c8-wasmsimd.c.in
index 51dc5cb..77282fc 100644
--- a/src/qs8-igemm/MRx4c8-wasmsimd.c.in
+++ b/src/qs8-igemm/MRx4c8-wasmsimd.c.in
@@ -10,6 +10,7 @@
#include <wasm_simd128.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
diff --git a/src/qs8-igemm/MRx8c8-avx2.c.in b/src/qs8-igemm/MRx8c8-avx2.c.in
index f30a5e8..ed18162 100644
--- a/src/qs8-igemm/MRx8c8-avx2.c.in
+++ b/src/qs8-igemm/MRx8c8-avx2.c.in
@@ -10,6 +10,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_${MR}x8c8__avx2(
@@ -37,6 +38,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
diff --git a/src/qs8-igemm/MRxNRc4-neondot.c.in b/src/qs8-igemm/MRxNRc4-neondot.c.in
index 00b0382..e49b634 100644
--- a/src/qs8-igemm/MRxNRc4-neondot.c.in
+++ b/src/qs8-igemm/MRxNRc4-neondot.c.in
@@ -10,8 +10,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_${MR}x${NR}c4__neondot(
@@ -39,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
@@ -92,7 +93,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a ${MR}x4 block of activations.
$for M in range(MR):
diff --git a/src/qs8-igemm/c16-neon-mlal-padal.c.in b/src/qs8-igemm/c16-neon-mlal-padal.c.in
index f2202a4..1d95ae3 100644
--- a/src/qs8-igemm/c16-neon-mlal-padal.c.in
+++ b/src/qs8-igemm/c16-neon-mlal-padal.c.in
@@ -10,8 +10,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_${MR}x${NR}c16__neon_mlal_padal(
@@ -39,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
diff --git a/src/qs8-igemm/c2-neon-mull-padal-dup.c.in b/src/qs8-igemm/c2-neon-mull-padal-dup.c.in
index 08dc2a0..79e350b 100644
--- a/src/qs8-igemm/c2-neon-mull-padal-dup.c.in
+++ b/src/qs8-igemm/c2-neon-mull-padal-dup.c.in
@@ -10,8 +10,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_${MR}x${NR}c2__neon_${"mlal" if MLA else "mull"}_padal_dup(
@@ -39,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
@@ -143,16 +144,6 @@
$for N in range(0, NR, 4):
const int16x8_t vprod${M}x${ABC[N:N+4]}c2 = vmull_s8(vb${ABC[N:N+4]}c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va${M}), 2)));
vacc${M}x${ABC[N:N+4]} = vpadalq_s16(vacc${M}x${ABC[N:N+4]}, vprod${M}x${ABC[N:N+4]}c2);
-
- if (k > 6 * sizeof(int8_t)) {
- $for N in range(0, NR, 4):
- const int8x8_t vb${ABC[N:N+4]}c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- $for M in range(MR):
- $for N in range(0, NR, 4):
- const int16x8_t vprod${M}x${ABC[N:N+4]}c3 = vmull_s8(vb${ABC[N:N+4]}c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va${M}), 3)));
- vacc${M}x${ABC[N:N+4]} = vpadalq_s16(vacc${M}x${ABC[N:N+4]}, vprod${M}x${ABC[N:N+4]}c3);
- }
}
}
}
diff --git a/src/qs8-igemm/c8-neon-mull-padal.c.in b/src/qs8-igemm/c8-neon-mull-padal.c.in
index 26c8336..66b17b1 100644
--- a/src/qs8-igemm/c8-neon-mull-padal.c.in
+++ b/src/qs8-igemm/c8-neon-mull-padal.c.in
@@ -10,8 +10,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_${MR}x${NR}c8__neon_mull_padal(
@@ -39,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
$for M in range(1, MR):
int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
@@ -93,9 +94,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
$for M in range(MR):
const int8x8_t va${M} = vld1_s8(a${M});
diff --git a/src/qs8-igemm/gen/12x8c4-minmax-neondot.c b/src/qs8-igemm/gen/12x8c4-minmax-neondot.c
index d6c5eca..7a74253 100644
--- a/src/qs8-igemm/gen/12x8c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/12x8c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -240,7 +241,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 12x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/1x16c16-minmax-neon-mlal-padal.c b/src/qs8-igemm/gen/1x16c16-minmax-neon-mlal-padal.c
index 293022a..302733a 100644
--- a/src/qs8-igemm/gen/1x16c16-minmax-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/1x16c16-minmax-neon-mlal-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x16c16__neon_mlal_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x16c2-minmax-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/1x16c2-minmax-neon-mlal-padal-dup.c
index 21f0a45..84cc0e7 100644
--- a/src/qs8-igemm/gen/1x16c2-minmax-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/1x16c2-minmax-neon-mlal-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x16c2__neon_mlal_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -249,22 +250,6 @@
vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc2);
const int16x8_t vprod0xCDEFc2 = vmull_s8(vbCDEFc2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2)));
vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
- const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x16c2-minmax-neon-mull-padal-dup.c b/src/qs8-igemm/gen/1x16c2-minmax-neon-mull-padal-dup.c
index f384462..9ae6fc7 100644
--- a/src/qs8-igemm/gen/1x16c2-minmax-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/1x16c2-minmax-neon-mull-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x16c2__neon_mull_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -161,22 +162,6 @@
vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc2);
const int16x8_t vprod0xCDEFc2 = vmull_s8(vbCDEFc2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2)));
vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
- const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x16c4-minmax-neondot.c b/src/qs8-igemm/gen/1x16c4-minmax-neondot.c
index fb5c186..a72ae5b 100644
--- a/src/qs8-igemm/gen/1x16c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/1x16c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x16c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
do {
@@ -85,7 +86,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 1x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
index 924734f..34e6a19 100644
--- a/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
@@ -13,6 +13,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x16c8__avx512skx(
@@ -38,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
diff --git a/src/qs8-igemm/gen/1x16c8-minmax-neon-mull-padal.c b/src/qs8-igemm/gen/1x16c8-minmax-neon-mull-padal.c
index ca428ce..811cb56 100644
--- a/src/qs8-igemm/gen/1x16c8-minmax-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/1x16c8-minmax-neon-mull-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x16c8__neon_mull_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
@@ -159,9 +160,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
const int8x8_t va0 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
index 42dc29e..cdff4d5 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c2__sse2_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -110,15 +112,6 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
index d78c425..bd6ce42 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c2__sse2_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -110,15 +112,6 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
index 0d5b1dc..312afa9 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c2__sse41_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -110,15 +112,6 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
index 2827d03..af2b116 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c2__sse41_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -110,15 +112,6 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
index b9aba97..5f308dd 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c2__ssse3_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -110,15 +112,6 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
index 862a3f6..ff8241a 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c2__ssse3_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -110,15 +112,6 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
index 5882450..fe4d345 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c2__xop_ld128(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -115,15 +117,6 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
index 5aafdf0..7417f8e 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c2__xop_ld64(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -115,15 +117,6 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
index fe9eb8c..6837212 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__sse2_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
index 2dd1d31..fc91529 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__sse2_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
index 10b9d02..66f8bbb 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__sse41_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
index c171ead..4031b08 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__sse41_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
index 561d72f..b04f5bf 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__ssse3_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
index f95561d..6036ab6 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__ssse3_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-wasmsimd-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-wasmsimd-ld128.c
index 8a104c0..9af046a 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-wasmsimd-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-wasmsimd-ld128.c
@@ -12,6 +12,7 @@
#include <wasm_simd128.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__wasmsimd_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
const v128_t vzero = wasm_f64x2_splat(0.0);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-wasmsimd-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-wasmsimd-ld64.c
index 15bc8bb..faadba9 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-wasmsimd-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-wasmsimd-ld64.c
@@ -12,6 +12,7 @@
#include <wasm_simd128.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__wasmsimd_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
const v128_t vzero = wasm_f64x2_splat(0.0);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
index 0686d4e..aa7eca9 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__xop_ld128(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
index fb46c83..1204750 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x4c8__xop_ld64(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x8c16-minmax-neon-mlal-padal.c b/src/qs8-igemm/gen/1x8c16-minmax-neon-mlal-padal.c
index de8a413..e81fbe7 100644
--- a/src/qs8-igemm/gen/1x8c16-minmax-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/1x8c16-minmax-neon-mlal-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x8c16__neon_mlal_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x8c2-minmax-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/1x8c2-minmax-neon-mlal-padal-dup.c
index 0f4c1cf..ccd894e 100644
--- a/src/qs8-igemm/gen/1x8c2-minmax-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/1x8c2-minmax-neon-mlal-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x8c2__neon_mlal_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -165,16 +166,6 @@
vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);
const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2)));
vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x8c2-minmax-neon-mull-padal-dup.c b/src/qs8-igemm/gen/1x8c2-minmax-neon-mull-padal-dup.c
index d8d8367..c2f45d1 100644
--- a/src/qs8-igemm/gen/1x8c2-minmax-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/1x8c2-minmax-neon-mull-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x8c2__neon_mull_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
do {
@@ -117,16 +118,6 @@
vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);
const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2)));
vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/1x8c4-minmax-neondot.c b/src/qs8-igemm/gen/1x8c4-minmax-neondot.c
index 07c1ad1..ed42605 100644
--- a/src/qs8-igemm/gen/1x8c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/1x8c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x8c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
do {
@@ -75,7 +76,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 1x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-avx2.c b/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
index 4eaa400..4c41935 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
@@ -13,6 +13,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x8c8__avx2(
@@ -40,6 +41,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-neon-mull-padal.c b/src/qs8-igemm/gen/1x8c8-minmax-neon-mull-padal.c
index 86e4775..d4c86c9 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-neon-mull-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_1x8c8__neon_mull_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
do {
@@ -111,9 +112,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
const int8x8_t va0 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/2x16c16-minmax-neon-mlal-padal.c b/src/qs8-igemm/gen/2x16c16-minmax-neon-mlal-padal.c
index 293e5e5..db2455b 100644
--- a/src/qs8-igemm/gen/2x16c16-minmax-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/2x16c16-minmax-neon-mlal-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x16c16__neon_mlal_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x16c2-minmax-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/2x16c2-minmax-neon-mlal-padal-dup.c
index 8fa5b3c..d35ce3f 100644
--- a/src/qs8-igemm/gen/2x16c2-minmax-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/2x16c2-minmax-neon-mlal-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x16c2__neon_mlal_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
@@ -369,30 +370,6 @@
vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc2);
const int16x8_t vprod1xCDEFc2 = vmull_s8(vbCDEFc2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2)));
vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
- const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);
- const int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/2x16c2-minmax-neon-mull-padal-dup.c b/src/qs8-igemm/gen/2x16c2-minmax-neon-mull-padal-dup.c
index 867c6c0..b13ce66 100644
--- a/src/qs8-igemm/gen/2x16c2-minmax-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/2x16c2-minmax-neon-mull-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x16c2__neon_mull_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
@@ -231,30 +232,6 @@
vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc2);
const int16x8_t vprod1xCDEFc2 = vmull_s8(vbCDEFc2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2)));
vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
- const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);
- const int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
index 347ed78..328bc55 100644
--- a/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
@@ -13,6 +13,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x16c8__avx512skx(
@@ -38,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x16c8-minmax-neon-mull-padal.c b/src/qs8-igemm/gen/2x16c8-minmax-neon-mull-padal.c
index 2595047..c6cd3ef 100644
--- a/src/qs8-igemm/gen/2x16c8-minmax-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/2x16c8-minmax-neon-mull-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x16c8__neon_mull_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
@@ -233,9 +234,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
const int8x8_t va0 = vld1_s8(a0);
const int8x8_t va1 = vld1_s8(a1);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
index 906861e..c771d14 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__sse2_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
index 3a728bd..e49603b 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__sse2_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
index 6d68d36..e5e18fd 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__sse41_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
index c7b7392..09d8a27 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__sse41_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
index 2b1d86a..d10dc86 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__ssse3_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
index 15e19f9..2b97a3c 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__ssse3_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-wasmsimd-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-wasmsimd-ld128.c
index 11188d4..3aa0434 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-wasmsimd-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-wasmsimd-ld128.c
@@ -12,6 +12,7 @@
#include <wasm_simd128.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__wasmsimd_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-wasmsimd-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-wasmsimd-ld64.c
index a85d4d7..d2675e6 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-wasmsimd-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-wasmsimd-ld64.c
@@ -12,6 +12,7 @@
#include <wasm_simd128.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__wasmsimd_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
index 00d6188..e58f425 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__xop_ld128(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
index 70458bd..02bbdf4 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x4c8__xop_ld64(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x8c16-minmax-neon-mlal-padal.c b/src/qs8-igemm/gen/2x8c16-minmax-neon-mlal-padal.c
index c95128a..bc9ba0e 100644
--- a/src/qs8-igemm/gen/2x8c16-minmax-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/2x8c16-minmax-neon-mlal-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x8c16__neon_mlal_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x8c2-minmax-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/2x8c2-minmax-neon-mlal-padal-dup.c
index 478e2c5..f48331d 100644
--- a/src/qs8-igemm/gen/2x8c2-minmax-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/2x8c2-minmax-neon-mlal-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x8c2__neon_mlal_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
@@ -231,20 +232,6 @@
vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);
const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2)));
vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/2x8c2-minmax-neon-mull-padal-dup.c b/src/qs8-igemm/gen/2x8c2-minmax-neon-mull-padal-dup.c
index 03fa52d..5178132 100644
--- a/src/qs8-igemm/gen/2x8c2-minmax-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/2x8c2-minmax-neon-mull-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x8c2__neon_mull_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
@@ -157,20 +158,6 @@
vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);
const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2)));
vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-avx2.c b/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
index d3060c3..9291845 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
@@ -13,6 +13,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x8c8__avx2(
@@ -40,6 +41,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-neon-mull-padal.c b/src/qs8-igemm/gen/2x8c8-minmax-neon-mull-padal.c
index 9da18a4..875fe8e 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-neon-mull-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_2x8c8__neon_mull_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr != 2) {
@@ -153,9 +154,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
const int8x8_t va0 = vld1_s8(a0);
const int8x8_t va1 = vld1_s8(a1);
diff --git a/src/qs8-igemm/gen/3x16c16-minmax-neon-mlal-padal.c b/src/qs8-igemm/gen/3x16c16-minmax-neon-mlal-padal.c
index 70f26b2..2b9bfae 100644
--- a/src/qs8-igemm/gen/3x16c16-minmax-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/3x16c16-minmax-neon-mlal-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x16c16__neon_mlal_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x16c2-minmax-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/3x16c2-minmax-neon-mlal-padal-dup.c
index 6d304e2..5fa8f78 100644
--- a/src/qs8-igemm/gen/3x16c2-minmax-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/3x16c2-minmax-neon-mlal-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x16c2__neon_mlal_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -489,38 +490,6 @@
vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc2);
const int16x8_t vprod2xCDEFc2 = vmull_s8(vbCDEFc2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 2)));
vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
- const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);
- const int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);
- const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
- const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
- const int16x8_t vprod2x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc3);
- const int16x8_t vprod2xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/3x16c2-minmax-neon-mull-padal-dup.c b/src/qs8-igemm/gen/3x16c2-minmax-neon-mull-padal-dup.c
index 53161f3..c971f66 100644
--- a/src/qs8-igemm/gen/3x16c2-minmax-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/3x16c2-minmax-neon-mull-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x16c2__neon_mull_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -301,38 +302,6 @@
vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc2);
const int16x8_t vprod2xCDEFc2 = vmull_s8(vbCDEFc2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 2)));
vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
- const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);
- const int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);
- const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
- const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
- const int16x8_t vprod2x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc3);
- const int16x8_t vprod2xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
index 3fe63e4..f9708b7 100644
--- a/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
@@ -13,6 +13,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x16c8__avx512skx(
@@ -38,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x16c8-minmax-neon-mull-padal.c b/src/qs8-igemm/gen/3x16c8-minmax-neon-mull-padal.c
index 97fd9d7..9a305da 100644
--- a/src/qs8-igemm/gen/3x16c8-minmax-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/3x16c8-minmax-neon-mull-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x16c8__neon_mull_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -307,9 +308,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
const int8x8_t va0 = vld1_s8(a0);
const int8x8_t va1 = vld1_s8(a1);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
index dd940c0..47dba04 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__sse2_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
index 4904a1c..eaffe04 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__sse2_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
index 14d7f8c..b4c8d56 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__sse41_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
index 5450387..6092d91 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__sse41_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
index 3f3aefa..f428f76 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__ssse3_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
index 8cca813..141be70 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__ssse3_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-wasmsimd-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-wasmsimd-ld128.c
index 0c95932..f232658 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-wasmsimd-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-wasmsimd-ld128.c
@@ -12,6 +12,7 @@
#include <wasm_simd128.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__wasmsimd_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-wasmsimd-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-wasmsimd-ld64.c
index 7f22e3d..1b9efcc 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-wasmsimd-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-wasmsimd-ld64.c
@@ -12,6 +12,7 @@
#include <wasm_simd128.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__wasmsimd_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
index 1760fc0..4adc8c1 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__xop_ld128(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
index c215aa9..6bb8aaa 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x4c8__xop_ld64(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x8c16-minmax-neon-mlal-padal.c b/src/qs8-igemm/gen/3x8c16-minmax-neon-mlal-padal.c
index 17ddc35..626dfa4 100644
--- a/src/qs8-igemm/gen/3x8c16-minmax-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/3x8c16-minmax-neon-mlal-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x8c16__neon_mlal_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x8c2-minmax-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/3x8c2-minmax-neon-mlal-padal-dup.c
index fac3944..bb5b54f 100644
--- a/src/qs8-igemm/gen/3x8c2-minmax-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/3x8c2-minmax-neon-mlal-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x8c2__neon_mlal_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -297,24 +298,6 @@
vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);
const int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 2)));
vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
- const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/3x8c2-minmax-neon-mull-padal-dup.c b/src/qs8-igemm/gen/3x8c2-minmax-neon-mull-padal-dup.c
index 3f7f6b3..0177c18 100644
--- a/src/qs8-igemm/gen/3x8c2-minmax-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/3x8c2-minmax-neon-mull-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x8c2__neon_mull_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -197,24 +198,6 @@
vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);
const int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 2)));
vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
- const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-avx2.c b/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
index 4269041..231c884 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
@@ -13,6 +13,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x8c8__avx2(
@@ -40,6 +41,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-neon-mull-padal.c b/src/qs8-igemm/gen/3x8c8-minmax-neon-mull-padal.c
index 0e70a8b..5c7810d 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-neon-mull-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_3x8c8__neon_mull_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -195,9 +196,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
const int8x8_t va0 = vld1_s8(a0);
const int8x8_t va1 = vld1_s8(a1);
diff --git a/src/qs8-igemm/gen/4x16c16-minmax-neon-mlal-padal.c b/src/qs8-igemm/gen/4x16c16-minmax-neon-mlal-padal.c
index 8368cd2..df63cae 100644
--- a/src/qs8-igemm/gen/4x16c16-minmax-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/4x16c16-minmax-neon-mlal-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x16c16__neon_mlal_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/4x16c2-minmax-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/4x16c2-minmax-neon-mlal-padal-dup.c
index f6d24d2..ef26e6a 100644
--- a/src/qs8-igemm/gen/4x16c2-minmax-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/4x16c2-minmax-neon-mlal-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x16c2__neon_mlal_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -609,46 +610,6 @@
vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc2);
const int16x8_t vprod3xCDEFc2 = vmull_s8(vbCDEFc2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 2)));
vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
- const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);
- const int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);
- const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
- const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
- const int16x8_t vprod2x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc3);
- const int16x8_t vprod2xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc3);
- const int16x8_t vprod3x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c3);
- const int16x8_t vprod3x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c3);
- const int16x8_t vprod3x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc3);
- const int16x8_t vprod3xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x16c2-minmax-neon-mull-padal-dup.c b/src/qs8-igemm/gen/4x16c2-minmax-neon-mull-padal-dup.c
index d7a469b..6b8b47e 100644
--- a/src/qs8-igemm/gen/4x16c2-minmax-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/4x16c2-minmax-neon-mull-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x16c2__neon_mull_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -371,46 +372,6 @@
vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc2);
const int16x8_t vprod3xCDEFc2 = vmull_s8(vbCDEFc2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 2)));
vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
- const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);
- const int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);
- const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
- const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
- const int16x8_t vprod2x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc3);
- const int16x8_t vprod2xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc3);
- const int16x8_t vprod3x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c3);
- const int16x8_t vprod3x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c3);
- const int16x8_t vprod3x89ABc3 = vmull_s8(vb89ABc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc3);
- const int16x8_t vprod3xCDEFc3 = vmull_s8(vbCDEFc3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x16c4-minmax-neondot.c b/src/qs8-igemm/gen/4x16c4-minmax-neondot.c
index a315841..95dde6d 100644
--- a/src/qs8-igemm/gen/4x16c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/4x16c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x16c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -148,7 +149,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 4x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
index b16ed2c..4f724c4 100644
--- a/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
@@ -13,6 +13,7 @@
#include <xnnpack/igemm.h>
#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x16c8__avx512skx(
@@ -38,6 +39,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/4x16c8-minmax-neon-mull-padal.c b/src/qs8-igemm/gen/4x16c8-minmax-neon-mull-padal.c
index 9430f21..b03befc 100644
--- a/src/qs8-igemm/gen/4x16c8-minmax-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/4x16c8-minmax-neon-mull-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x16c8__neon_mull_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -381,9 +382,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
const int8x8_t va0 = vld1_s8(a0);
const int8x8_t va1 = vld1_s8(a1);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
index 89c7807..012a594 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x4c2__sse2_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -197,21 +199,6 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc1x0123 = _mm_add_epi32(vacc1x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc2x0123 = _mm_add_epi32(vacc2x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc3x0123 = _mm_add_epi32(vacc3x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
index 7b5ce73..3d83c87 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
@@ -12,6 +12,7 @@
#include <emmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x4c2__sse2_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -197,21 +199,6 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc1x0123 = _mm_add_epi32(vacc1x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc2x0123 = _mm_add_epi32(vacc2x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc3x0123 = _mm_add_epi32(vacc3x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
index 6fc97cc..d0a7c2a 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x4c2__sse41_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -197,21 +199,6 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc1x0123 = _mm_add_epi32(vacc1x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc2x0123 = _mm_add_epi32(vacc2x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc3x0123 = _mm_add_epi32(vacc3x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
index 4978b7b..9c34aa3 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
@@ -12,6 +12,7 @@
#include <smmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x4c2__sse41_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -197,21 +199,6 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc1x0123 = _mm_add_epi32(vacc1x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc2x0123 = _mm_add_epi32(vacc2x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc3x0123 = _mm_add_epi32(vacc3x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
index 82cfd1f..9c1304f 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x4c2__ssse3_ld128(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -197,21 +199,6 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc1x0123 = _mm_add_epi32(vacc1x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc2x0123 = _mm_add_epi32(vacc2x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc3x0123 = _mm_add_epi32(vacc3x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
index 8a2def1..2d56cdf 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
@@ -12,6 +12,7 @@
#include <tmmintrin.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x4c2__ssse3_ld64(
@@ -39,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -197,21 +199,6 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_add_epi32(vacc0x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc1x0123 = _mm_add_epi32(vacc1x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc2x0123 = _mm_add_epi32(vacc2x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- vacc3x0123 = _mm_add_epi32(vacc3x0123,
- _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
index b1b6880..e6faba2 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x4c2__xop_ld128(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -202,21 +204,6 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- vacc1x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- vacc2x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- vacc3x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
index 7f3c778..0253dfa 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
@@ -17,6 +17,7 @@
#endif
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x4c2__xop_ld64(
@@ -44,6 +45,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -202,21 +204,6 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
-
- if (k > 6 * sizeof(int8_t)) {
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
-
- vacc0x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- vacc1x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- vacc2x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- vacc3x0123 = _mm_maddd_epi16(
- _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x8c16-minmax-neon-mlal-padal.c b/src/qs8-igemm/gen/4x8c16-minmax-neon-mlal-padal.c
index 9486381..ef871fb 100644
--- a/src/qs8-igemm/gen/4x8c16-minmax-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/4x8c16-minmax-neon-mlal-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x8c16__neon_mlal_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 16);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
diff --git a/src/qs8-igemm/gen/4x8c2-minmax-neon-mlal-padal-dup.c b/src/qs8-igemm/gen/4x8c2-minmax-neon-mlal-padal-dup.c
index d135354..80aa9ac 100644
--- a/src/qs8-igemm/gen/4x8c2-minmax-neon-mlal-padal-dup.c
+++ b/src/qs8-igemm/gen/4x8c2-minmax-neon-mlal-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x8c2__neon_mlal_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -363,28 +364,6 @@
vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c2);
const int16x8_t vprod3x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 2)));
vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
- const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
- const int16x8_t vprod3x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c3);
- const int16x8_t vprod3x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x8c2-minmax-neon-mull-padal-dup.c b/src/qs8-igemm/gen/4x8c2-minmax-neon-mull-padal-dup.c
index 8afdcf0..5f635fa 100644
--- a/src/qs8-igemm/gen/4x8c2-minmax-neon-mull-padal-dup.c
+++ b/src/qs8-igemm/gen/4x8c2-minmax-neon-mull-padal-dup.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x8c2__neon_mull_padal_dup(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 2);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -237,28 +238,6 @@
vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c2);
const int16x8_t vprod3x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 2)));
vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c2);
-
- if (k > 6 * sizeof(int8_t)) {
- const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
- const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
-
- const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
- const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));
- vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
- const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
- const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));
- vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
- const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
- const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3)));
- vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
- const int16x8_t vprod3x0123c3 = vmull_s8(vb0123c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c3);
- const int16x8_t vprod3x4567c3 = vmull_s8(vb4567c3, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3)));
- vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c3);
- }
}
}
}
diff --git a/src/qs8-igemm/gen/4x8c4-minmax-neondot.c b/src/qs8-igemm/gen/4x8c4-minmax-neondot.c
index b7f6871..5758e21 100644
--- a/src/qs8-igemm/gen/4x8c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/4x8c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x8c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -120,7 +121,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 4x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/4x8c8-minmax-neon-mull-padal.c b/src/qs8-igemm/gen/4x8c8-minmax-neon-mull-padal.c
index 594adc9..ae06019 100644
--- a/src/qs8-igemm/gen/4x8c8-minmax-neon-mull-padal.c
+++ b/src/qs8-igemm/gen/4x8c8-minmax-neon-mull-padal.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_4x8c8__neon_mull_padal(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 8);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -237,9 +238,6 @@
k -= 16 * sizeof(int8_t);
}
// Handle up to 8 final positions of `k`
- // If kc was 0 or 16, there is no remainder. k is 0.
- // If kc was 1 to 8, there is a remainder of k.
- // If kc was 9 to 15, the main loop handled the remainder; k underflowed.
if XNN_UNLIKELY(k > 0) {
const int8x8_t va0 = vld1_s8(a0);
const int8x8_t va1 = vld1_s8(a1);
diff --git a/src/qs8-igemm/gen/6x16c4-minmax-neondot.c b/src/qs8-igemm/gen/6x16c4-minmax-neondot.c
index b013d4b..ab885b5 100644
--- a/src/qs8-igemm/gen/6x16c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/6x16c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_6x16c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -190,7 +191,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 6x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/6x8c4-minmax-neondot.c b/src/qs8-igemm/gen/6x8c4-minmax-neondot.c
index 13bb060..b4c3bf2 100644
--- a/src/qs8-igemm/gen/6x8c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/6x8c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_6x8c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -150,7 +151,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 6x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/8x16c4-minmax-neondot.c b/src/qs8-igemm/gen/8x16c4-minmax-neondot.c
index 40f8911..001ce33 100644
--- a/src/qs8-igemm/gen/8x16c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/8x16c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_8x16c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -232,7 +233,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 8x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);
diff --git a/src/qs8-igemm/gen/8x8c4-minmax-neondot.c b/src/qs8-igemm/gen/8x8c4-minmax-neondot.c
index d8f4223..9ce9937 100644
--- a/src/qs8-igemm/gen/8x8c4-minmax-neondot.c
+++ b/src/qs8-igemm/gen/8x8c4-minmax-neondot.c
@@ -11,8 +11,8 @@
#include <arm_neon.h>
-#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
void xnn_qs8_igemm_minmax_ukernel_8x8c4__neondot(
@@ -40,6 +40,7 @@
assert(w != NULL);
assert(c != NULL);
+ kc = round_up_po2(kc, 4);
int8_t* c0 = c;
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
@@ -180,7 +181,7 @@
k -= 8 * sizeof(int8_t);
}
- // Handle up to 7 final positions of `k`
+ // Handle up to 6 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 8x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0);