Support FP32 requantization in AVX2 QS8 microkernels
PiperOrigin-RevId: 375822588
diff --git a/src/qs8-gemm/MRx16c8-avx512skx.c.in b/src/qs8-gemm/MRx16c8-avx512skx.c.in
index 9b842bc..a2d725d 100644
--- a/src/qs8-gemm/MRx16c8-avx512skx.c.in
+++ b/src/qs8-gemm/MRx16c8-avx512skx.c.in
@@ -61,24 +61,24 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
$if MR > 1:
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
$else:
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
$if MR > 2:
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
$elif MR == 2:
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
$else:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
$for N in range(4, 16, 4):
diff --git a/src/qs8-gemm/MRx4c2-sse.c.in b/src/qs8-gemm/MRx4c2-sse.c.in
index 77306ed..cdd7e71 100644
--- a/src/qs8-gemm/MRx4c2-sse.c.in
+++ b/src/qs8-gemm/MRx4c2-sse.c.in
@@ -27,7 +27,7 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
+$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x4c2__${ISA}${LOAD_SUFFIX}(
size_t mr,
@@ -297,8 +297,8 @@
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -308,8 +308,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/MRx4c8-sse.c.in b/src/qs8-gemm/MRx4c8-sse.c.in
index 656f725..6f26c05 100644
--- a/src/qs8-gemm/MRx4c8-sse.c.in
+++ b/src/qs8-gemm/MRx4c8-sse.c.in
@@ -27,7 +27,7 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
+$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x4c8__${ISA}${LOAD_SUFFIX}(
size_t mr,
@@ -234,8 +234,8 @@
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -245,8 +245,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/MRx4c8-wasmsimd.c.in b/src/qs8-gemm/MRx4c8-wasmsimd.c.in
index c91c1cc..14ec68d 100644
--- a/src/qs8-gemm/MRx4c8-wasmsimd.c.in
+++ b/src/qs8-gemm/MRx4c8-wasmsimd.c.in
@@ -131,8 +131,8 @@
$for M in range(MR):
const v128_t vacc${M}x01 = wasm_v32x4_shuffle(vacc${M}x0123, vsign${M}x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
$for M in range(MR):
const v128_t vprod${M}x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc${M}x01, vmultiplier), vrounding);
const v128_t vacc${M}x23 = wasm_v32x4_shuffle(vacc${M}x0123, vsign${M}x0123, 2, 6, 3, 7);
@@ -143,16 +143,16 @@
$for M in range(MR):
const v128_t vq31prod${M}x0123 = wasm_v32x4_shuffle(vprod${M}x01, vprod${M}x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
$for M in range(MR):
const v128_t vrem${M}x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod${M}x0123, vremainder_mask), wasm_i32x4_lt(vq31prod${M}x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
$for M in range(MR):
vacc${M}x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod${M}x0123, vshift), wasm_i32x4_gt(vrem${M}x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
$for M in range(0, MR, 2):
v128_t vacc${M}${min(M+1, MR-1)}x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
@@ -161,10 +161,10 @@
$else:
v128_t vout = wasm_i8x16_narrow_i16x8(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/MRx8c8-avx2.c.in b/src/qs8-gemm/MRx8c8-avx2.c.in
index 761da0f..f92a9aa 100644
--- a/src/qs8-gemm/MRx8c8-avx2.c.in
+++ b/src/qs8-gemm/MRx8c8-avx2.c.in
@@ -3,6 +3,7 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD128", "EXTENDED"]
$assert MR <= 4
#include <assert.h>
@@ -15,7 +16,7 @@
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x8c8__avx2(
+void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x8c8__avx2(
size_t mr,
size_t nc,
size_t kc,
@@ -112,40 +113,51 @@
$for M in range(MR):
__m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
- $for M in range(MR):
- const __m256i vacc${M}x11335577 = _mm256_srli_epi64(vacc${M}x01234567, 32);
+ $for M in range(MR):
+ const __m256i vacc${M}x11335577 = _mm256_srli_epi64(vacc${M}x01234567, 32);
- $for M in range(MR):
- const __m256i vprod${M}x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x01234567, vmultiplier), vrounding);
+ $for M in range(MR):
+ const __m256i vprod${M}x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x01234567, vmultiplier), vrounding);
- $for M in range(MR):
- const __m256i vprod${M}x1357 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x11335577, vmultiplier), vrounding);
+ $for M in range(MR):
+ const __m256i vprod${M}x1357 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x11335577, vmultiplier), vrounding);
- $for M in range(MR):
- const __m256i vq31prod${M}x0246 = _mm256_srli_epi64(vprod${M}x0246, 31);
- const __m256i vq31prod${M}x1357 = _mm256_add_epi64(vprod${M}x1357, vprod${M}x1357);
+ $for M in range(MR):
+ const __m256i vq31prod${M}x0246 = _mm256_srli_epi64(vprod${M}x0246, 31);
+ const __m256i vq31prod${M}x1357 = _mm256_add_epi64(vprod${M}x1357, vprod${M}x1357);
- $for M in range(MR):
- const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
+ $for M in range(MR):
+ const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
- $for M in range(MR):
- const __m256i vrem${M}x01234567 =
- _mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
+ $for M in range(MR):
+ const __m256i vrem${M}x01234567 =
+ _mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
- $for M in range(MR):
- vacc${M}x01234567 =
- _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
+ $for M in range(MR):
+ vacc${M}x01234567 =
+ _mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m256 vscaled${M}x01234567 = _mm256_cvtepi32_ps(vacc${M}x01234567);
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ $for M in range(MR):
+ vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale);
+
+ $for M in range(MR):
+ vacc${M}x01234567 = _mm256_cvtps_epi32(vscaled${M}x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_zero_point);
$for M in range(0, MR, 2):
__m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point);
@@ -157,8 +169,8 @@
$else:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->${REQUANTIZATION.lower()}_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/MRxNRc4-neondot.c.in b/src/qs8-gemm/MRxNRc4-neondot.c.in
index d395508..eb3af38 100644
--- a/src/qs8-gemm/MRxNRc4-neondot.c.in
+++ b/src/qs8-gemm/MRxNRc4-neondot.c.in
@@ -105,12 +105,12 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
- const int32x4_t vproduct${M}x${ABC[N:N+4]} = vqrdmulhq_n_s32(vacc${M}x${ABC[N:N+4]}, params->neon.multiplier);
+ const int32x4_t vproduct${M}x${ABC[N:N+4]} = vqrdmulhq_n_s32(vacc${M}x${ABC[N:N+4]}, params->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -120,7 +120,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -149,11 +149,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/c16-neon-mlal-padal.c.in b/src/qs8-gemm/c16-neon-mlal-padal.c.in
index 7392300..ebfd029 100644
--- a/src/qs8-gemm/c16-neon-mlal-padal.c.in
+++ b/src/qs8-gemm/c16-neon-mlal-padal.c.in
@@ -104,12 +104,12 @@
int32x4_t vacc${M}x${ABC[N:N+4]} = vcombine_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]} );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -119,7 +119,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -147,11 +147,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/c2-neon-mull-padal-dup.c.in b/src/qs8-gemm/c2-neon-mull-padal-dup.c.in
index 2914567..54e9881 100644
--- a/src/qs8-gemm/c2-neon-mull-padal-dup.c.in
+++ b/src/qs8-gemm/c2-neon-mull-padal-dup.c.in
@@ -139,12 +139,12 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -154,7 +154,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -183,11 +183,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/c8-neon-mull-padal.c.in b/src/qs8-gemm/c8-neon-mull-padal.c.in
index da184fb..e704355 100644
--- a/src/qs8-gemm/c8-neon-mull-padal.c.in
+++ b/src/qs8-gemm/c8-neon-mull-padal.c.in
@@ -122,12 +122,12 @@
int32x4_t vacc${M}x${ABC[N:N+4]} = vcombine_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]} );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -137,7 +137,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -165,11 +165,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 231fdae..2a95279 100644
--- a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -224,13 +224,13 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -242,7 +242,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -254,8 +254,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
index c00e5f7..ab31654 100644
--- a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -222,13 +222,13 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -240,7 +240,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -252,8 +252,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
index b721ca4..9e9ed9c 100644
--- a/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/1x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -217,13 +217,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -235,7 +235,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -247,8 +247,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
index 3a58012..42751c2 100644
--- a/src/qs8-gemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -176,13 +176,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -194,7 +194,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -205,8 +205,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index f76e520..465927a 100644
--- a/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -242,13 +242,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -260,7 +260,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -272,8 +272,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index df0e6fb..50e5fc7 100644
--- a/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/1x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -154,13 +154,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -172,7 +172,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -184,8 +184,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/1x16c4-minmax-gemmlowp-neondot.c
index 1d8d2aa..f0bcc60 100644
--- a/src/qs8-gemm/gen/1x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/1x16c4-minmax-gemmlowp-neondot.c
@@ -97,13 +97,13 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->neon.multiplier);
- const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -115,7 +115,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -127,8 +127,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
index a79b986..9eb5ad8 100644
--- a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
@@ -43,14 +43,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
index ff36aa9..3609135 100644
--- a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -249,13 +249,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -267,7 +267,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -278,8 +278,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
index ed1d96d..e1c439d 100644
--- a/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -160,13 +160,13 @@
int32x4_t vacc0xCDEF = vcombine_s32(vsum0xCD, vsum0xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
vacc0xCDEF = vqrdmulhq_s32(vacc0xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -178,7 +178,7 @@
vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_shift);
vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -189,8 +189,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/1x2-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/1x2-minmax-gemmlowp-scalar.c
index 65fa4d8..7a91fad 100644
--- a/src/qs8-gemm/gen/1x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/1x2-minmax-gemmlowp-scalar.c
@@ -52,7 +52,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
@@ -60,24 +60,24 @@
const int32_t vq31product0x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x0 + vq31rounding) >> 31);
const int32_t vq31product0x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/1x4-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/1x4-minmax-gemmlowp-scalar.c
index dbd2cd8..31866f0 100644
--- a/src/qs8-gemm/gen/1x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/1x4-minmax-gemmlowp-scalar.c
@@ -58,7 +58,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -70,32 +70,32 @@
const int32_t vq31product0x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x2 + vq31rounding) >> 31);
const int32_t vq31product0x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct0x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
const int32_t vremainder0x3 = (vq31product0x3 & vremainder_mask) - (int32_t) (vq31product0x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
int32_t vout0x3 = asr_s32(vq31product0x3, vshift) + (int32_t) (vremainder0x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
vout0x3 = math_max_s32(vout0x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
vout0x3 = math_min_s32(vout0x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
index abdb94d..427cb9f 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,23 +119,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
index 4acc0f6..b1c30ad 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,23 +119,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
index 1ae6dca..1836237 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -135,20 +135,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
index ad654f3..4d16fd0 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -135,20 +135,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
index e630e68..e97a400 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,23 +119,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
index 3360b16..278ad8f 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,23 +119,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
index 9eae431..f9cb7c7 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -135,20 +135,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
index 314f342..6e7a449 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -135,20 +135,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
index 5ef3e18..21adf15 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
@@ -110,8 +110,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,23 +124,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
index fec0dce..0b07fc7 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
@@ -110,8 +110,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,23 +124,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
index cfa7d99..3fdd41b 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -112,23 +112,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
index 18399ef..37bf918 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -128,20 +128,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
index 7895ded..ca397ec 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -112,23 +112,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
index 875056a..946b93d 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -128,20 +128,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
index 1b212df..543f507 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
@@ -103,8 +103,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,23 +117,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
index fc4d147..993b2e4 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -91,23 +91,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
index 95e0802..c6f6930 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -93,23 +93,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
index 414f82d..182a674 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -107,20 +107,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
index bd8a13a..3418021 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -109,20 +109,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
index 6aaef3f..8d72bbb 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -91,23 +91,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
index 6ff1e07..e63746f 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -93,23 +93,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
index a885ec0..a54757a 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -107,20 +107,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
index c502eb5..bf5e58a 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -109,20 +109,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index c4af47a..a452a7c 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -91,8 +91,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
@@ -100,22 +100,22 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc00x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc0x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc00x0123, vacc00x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index f97e401..1566b89 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -87,8 +87,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
@@ -96,22 +96,22 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc00x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc0x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc00x0123, vacc00x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
index 26310c8..0809765 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
@@ -82,8 +82,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -96,23 +96,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
index 393e07d..f6399da 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
@@ -84,8 +84,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -98,23 +98,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
index ec0c611..3175475 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -89,23 +89,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
index 863f22e..ddd443e 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -105,20 +105,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
index 23dee4c..1e07d0d 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -89,23 +89,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
index 3ba2f0d..04ec0c9 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
@@ -105,20 +105,20 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-wasmsimd.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-wasmsimd.c
index be88364..a093a7e 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-wasmsimd.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-wasmsimd.c
@@ -87,8 +87,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
@@ -96,22 +96,22 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc00x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc0x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc00x0123, vacc00x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
index 8766948..674b44b 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
@@ -80,8 +80,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -94,23 +94,23 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 7329a94..3ad4613 100644
--- a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -153,11 +153,11 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -165,7 +165,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -175,8 +175,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
index f3a68a8..62da1a8 100644
--- a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -152,11 +152,11 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -164,7 +164,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -174,8 +174,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
index a5a8fd6..faf07de 100644
--- a/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/1x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -147,11 +147,11 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -159,7 +159,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -169,8 +169,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 6e110e0..4fbc6cf 100644
--- a/src/qs8-gemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -116,11 +116,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -128,7 +128,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -137,8 +137,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 1944ffe..86a22df 100644
--- a/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -158,11 +158,11 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -170,7 +170,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -180,8 +180,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 822f923..f73c463 100644
--- a/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/1x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -110,11 +110,11 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -122,7 +122,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -132,8 +132,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/1x8c4-minmax-gemmlowp-neondot.c
index 3e12c01..bec1770 100644
--- a/src/qs8-gemm/gen/1x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/1x8c4-minmax-gemmlowp-neondot.c
@@ -83,11 +83,11 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -95,7 +95,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
@@ -105,8 +105,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..9c1b57c
--- /dev/null
+++ b/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
+
+ vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
index 17dcb23..e1761aa 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
@@ -91,8 +91,8 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
@@ -105,24 +105,24 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
index f8783fb..85b5d1b 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -157,11 +157,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -169,7 +169,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -178,8 +178,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
index 1c010f1..890cecf 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -108,11 +108,11 @@
int32x4_t vacc0x4567 = vcombine_s32(vsum0x45, vsum0x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -120,7 +120,7 @@
vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift);
vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
@@ -129,8 +129,8 @@
int8x8_t vout0x01234567 = vqmovn_s16(vacc0x01234567);
#endif
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567 = vmax_s8(vout0x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
index 622083f..be4ec37 100644
--- a/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
@@ -87,8 +87,8 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
@@ -101,24 +101,24 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 1c940b0..45cb137 100644
--- a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -298,7 +298,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -308,7 +308,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -328,7 +328,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -346,8 +346,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
index 7e645ff..2b529c9 100644
--- a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -296,7 +296,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -306,7 +306,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -326,7 +326,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -344,8 +344,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
index 4848f85..c51a42b 100644
--- a/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/2x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -319,7 +319,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -329,7 +329,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -349,7 +349,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -367,8 +367,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
index b7b984a..7409287 100644
--- a/src/qs8-gemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -287,7 +287,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -297,7 +297,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -317,7 +317,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -334,8 +334,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index afae933..209dd15 100644
--- a/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -360,7 +360,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -370,7 +370,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -390,7 +390,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -408,8 +408,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 9e1bac1..34e8361 100644
--- a/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/2x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -222,7 +222,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -232,7 +232,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -252,7 +252,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -270,8 +270,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
index 5b4ea8c..447dc07 100644
--- a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-avx512skx.c
@@ -49,14 +49,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
index e0b9794..716c0c3 100644
--- a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -394,7 +394,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -404,7 +404,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -424,7 +424,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -441,8 +441,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
index 4d865b4..dbdaecc 100644
--- a/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/2x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -255,7 +255,7 @@
int32x4_t vacc1xCDEF = vcombine_s32(vsum1xCD, vsum1xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -265,7 +265,7 @@
vacc1x89AB = vqrdmulhq_s32(vacc1x89AB, vmultiplier);
vacc1xCDEF = vqrdmulhq_s32(vacc1xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -285,7 +285,7 @@
vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_shift);
vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -302,8 +302,8 @@
int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/2x2-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/2x2-minmax-gemmlowp-scalar.c
index 93d7387..9f1c433 100644
--- a/src/qs8-gemm/gen/2x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/2x2-minmax-gemmlowp-scalar.c
@@ -63,7 +63,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -75,32 +75,32 @@
const int32_t vq31product1x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x0 + vq31rounding) >> 31);
const int32_t vq31product1x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
const int32_t vremainder1x1 = (vq31product1x1 & vremainder_mask) - (int32_t) (vq31product1x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
int32_t vout1x1 = asr_s32(vq31product1x1, vshift) + (int32_t) (vremainder1x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
vout1x1 = math_max_s32(vout1x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
vout1x1 = math_min_s32(vout1x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/2x4-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/2x4-minmax-gemmlowp-scalar.c
index 7f67584..29d8508 100644
--- a/src/qs8-gemm/gen/2x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/2x4-minmax-gemmlowp-scalar.c
@@ -73,7 +73,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -93,7 +93,7 @@
const int32_t vq31product1x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x2 + vq31rounding) >> 31);
const int32_t vq31product1x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct1x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -103,8 +103,8 @@
const int32_t vremainder1x2 = (vq31product1x2 & vremainder_mask) - (int32_t) (vq31product1x2 < 0);
const int32_t vremainder1x3 = (vq31product1x3 & vremainder_mask) - (int32_t) (vq31product1x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -114,7 +114,7 @@
int32_t vout1x2 = asr_s32(vq31product1x2, vshift) + (int32_t) (vremainder1x2 > vremainder_threshold);
int32_t vout1x3 = asr_s32(vq31product1x3, vshift) + (int32_t) (vremainder1x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -124,7 +124,7 @@
vout1x2 = math_max_s32(vout1x2, vout_min);
vout1x3 = math_max_s32(vout1x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -134,7 +134,7 @@
vout1x2 = math_min_s32(vout1x2, vout_max);
vout1x3 = math_min_s32(vout1x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
index 5c8c65f..de0580f 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,27 +152,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
index 73d5422..4213ba5 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,27 +152,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
index b5be8c7..3314380 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -176,24 +176,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
index 453aceb..3e0a980 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -176,24 +176,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
index 1c31e16..e59f622 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,27 +152,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
index b75bcde..5bbf9c3 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,27 +152,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
index 3b732b8..a1eb413 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -176,24 +176,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
index f177ef9..d917ca9 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -176,24 +176,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
index 6f3dae5..42fec23 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
@@ -137,8 +137,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -157,27 +157,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
index 04f48d1..f225098 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
@@ -137,8 +137,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -157,27 +157,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
index 6d70934..1f998e1 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,27 +145,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
index f040a7b..ea6a894 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -169,24 +169,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
index 6d6fd5a..ff5840a 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,27 +145,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
index 8f67268..895b0cf 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -169,24 +169,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
index cf0622b..4bc8c55 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
@@ -130,8 +130,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -150,27 +150,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
index 26c20db..df5de10 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,27 +117,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
index 9549f20..ae85181 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,27 +119,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
index 6afe7f8..85f2c10 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -141,24 +141,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
index aa2c3f8..9813058 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -143,24 +143,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
index 063b30a..1691be1 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,27 +117,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
index 086af13..c619ae7 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,27 +119,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
index 663f448..bd7bd88 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -141,24 +141,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
index 8e5eca9..2928523 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -143,24 +143,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index eccd1b3..085e9a9 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -120,8 +120,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -133,24 +133,24 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc01x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index 35141ec..83fdd00 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -116,8 +116,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -129,24 +129,24 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc01x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
index d1d91f5..6903e30 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
@@ -102,8 +102,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -122,27 +122,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
index 42f3678..c38e481 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
@@ -104,8 +104,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,27 +124,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
index f4f8c0b..86f9c1d 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -115,27 +115,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
index 344df2a..8b67fcd 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -139,24 +139,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
index 5afe66e..ea4d124 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -115,27 +115,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
index 5985c84..54d9ab4 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -139,24 +139,24 @@
const __m128i vq31prod0x0123 = _mm_shuffle_epi32(vq31prod0x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-wasmsimd.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-wasmsimd.c
index 02ed0f8..50d00a3 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-wasmsimd.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-wasmsimd.c
@@ -116,8 +116,8 @@
const v128_t vacc0x01 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 0, 4, 1, 5);
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -129,24 +129,24 @@
const v128_t vq31prod0x0123 = wasm_v32x4_shuffle(vprod0x01, vprod0x23, 1, 3, 5, 7);
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc01x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
index 6044996..a22b257 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
@@ -100,8 +100,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -120,27 +120,27 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 87f0291..8208fa0 100644
--- a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -195,13 +195,13 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -213,7 +213,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -225,8 +225,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
index 687a9b3..c5e0cdb 100644
--- a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -194,13 +194,13 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -212,7 +212,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -224,8 +224,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
index de05fc5..1df366c 100644
--- a/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/2x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -202,13 +202,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -220,7 +220,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -232,8 +232,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 64eafde..9e6ecc3 100644
--- a/src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -175,13 +175,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -193,7 +193,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -204,8 +204,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 8531b93..0572c92 100644
--- a/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -222,13 +222,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -240,7 +240,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -252,8 +252,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index de48d33..9741b73 100644
--- a/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/2x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -148,13 +148,13 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -166,7 +166,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -178,8 +178,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..d63d4f9
--- /dev/null
+++ b/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,178 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+ _mm_storeu_si32(c1, vout_hi);
+
+ c0 += 4;
+ c1 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+
+ c0 += 2;
+ c1 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
index d388284..1e39921 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
@@ -112,8 +112,8 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
@@ -132,28 +132,28 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 923d485..6d45009 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -234,13 +234,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -252,7 +252,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -263,8 +263,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
index e05b02b..612c0bc 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -159,13 +159,13 @@
int32x4_t vacc1x4567 = vcombine_s32(vsum1x45, vsum1x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -177,7 +177,7 @@
vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift);
vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -188,8 +188,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
index 6c39642..810bae4 100644
--- a/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
@@ -108,8 +108,8 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
@@ -128,28 +128,28 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 0c56b19..7729647 100644
--- a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -372,7 +372,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -386,7 +386,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -414,7 +414,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -438,8 +438,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
index 58deb96..b95411bf 100644
--- a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -370,7 +370,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -384,7 +384,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -412,7 +412,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -436,8 +436,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
index b17e441..55436f4 100644
--- a/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/3x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -421,7 +421,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -435,7 +435,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -463,7 +463,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -487,8 +487,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
index e57ca14..3063ac0 100644
--- a/src/qs8-gemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/3x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -398,7 +398,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -412,7 +412,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -440,7 +440,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -463,8 +463,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index ce9b048..dca07ae 100644
--- a/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -478,7 +478,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -492,7 +492,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -520,7 +520,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -544,8 +544,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index b021ba0..01be9bb 100644
--- a/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/3x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -290,7 +290,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -304,7 +304,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -332,7 +332,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -356,8 +356,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
index e94d74c..be2d1b5 100644
--- a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-avx512skx.c
@@ -55,14 +55,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
index 60be3af..b9aa38f 100644
--- a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -539,7 +539,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -553,7 +553,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -581,7 +581,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -604,8 +604,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
index ca496b1..d328f83 100644
--- a/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/3x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -350,7 +350,7 @@
int32x4_t vacc2xCDEF = vcombine_s32(vsum2xCD, vsum2xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -364,7 +364,7 @@
vacc2x89AB = vqrdmulhq_s32(vacc2x89AB, vmultiplier);
vacc2xCDEF = vqrdmulhq_s32(vacc2xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -392,7 +392,7 @@
vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_shift);
vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -415,8 +415,8 @@
int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/3x2-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/3x2-minmax-gemmlowp-scalar.c
index 10a266a..5b9009b 100644
--- a/src/qs8-gemm/gen/3x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/3x2-minmax-gemmlowp-scalar.c
@@ -74,7 +74,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -90,7 +90,7 @@
const int32_t vq31product2x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x0 + vq31rounding) >> 31);
const int32_t vq31product2x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
@@ -98,8 +98,8 @@
const int32_t vremainder2x0 = (vq31product2x0 & vremainder_mask) - (int32_t) (vq31product2x0 < 0);
const int32_t vremainder2x1 = (vq31product2x1 & vremainder_mask) - (int32_t) (vq31product2x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
@@ -107,7 +107,7 @@
int32_t vout2x0 = asr_s32(vq31product2x0, vshift) + (int32_t) (vremainder2x0 > vremainder_threshold);
int32_t vout2x1 = asr_s32(vq31product2x1, vshift) + (int32_t) (vremainder2x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
@@ -115,7 +115,7 @@
vout2x0 = math_max_s32(vout2x0, vout_min);
vout2x1 = math_max_s32(vout2x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
@@ -123,7 +123,7 @@
vout2x0 = math_min_s32(vout2x0, vout_max);
vout2x1 = math_min_s32(vout2x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/3x4-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/3x4-minmax-gemmlowp-scalar.c
index 61c64a9..1ae2a6b 100644
--- a/src/qs8-gemm/gen/3x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/3x4-minmax-gemmlowp-scalar.c
@@ -88,7 +88,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -116,7 +116,7 @@
const int32_t vq31product2x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x2 + vq31rounding) >> 31);
const int32_t vq31product2x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct2x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -130,8 +130,8 @@
const int32_t vremainder2x2 = (vq31product2x2 & vremainder_mask) - (int32_t) (vq31product2x2 < 0);
const int32_t vremainder2x3 = (vq31product2x3 & vremainder_mask) - (int32_t) (vq31product2x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -145,7 +145,7 @@
int32_t vout2x2 = asr_s32(vq31product2x2, vshift) + (int32_t) (vremainder2x2 > vremainder_threshold);
int32_t vout2x3 = asr_s32(vq31product2x3, vshift) + (int32_t) (vremainder2x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -159,7 +159,7 @@
vout2x2 = math_max_s32(vout2x2, vout_min);
vout2x3 = math_max_s32(vout2x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -173,7 +173,7 @@
vout2x2 = math_min_s32(vout2x2, vout_max);
vout2x3 = math_min_s32(vout2x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
index 885766e..acf1eb3 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,15 +202,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
index 99e227f..1d059b7 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,15 +202,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
index 56635b9..8cc7035 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -217,7 +217,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -225,8 +225,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -234,12 +234,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
index 39e555d..a01a80a 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -217,7 +217,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -225,8 +225,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -234,12 +234,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
index e106ae8..a6a16fa 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,15 +202,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
index 18c00d6..c60f805 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,15 +202,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
index c89d3bc..b693059 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -217,7 +217,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -225,8 +225,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -234,12 +234,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
index d033069..7193059 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -217,7 +217,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -225,8 +225,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -234,12 +234,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
index 49e37b1..525d542 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
@@ -164,8 +164,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -190,7 +190,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -198,8 +198,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -207,15 +207,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
index bb6ea3a..9fb43b5 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
@@ -164,8 +164,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -190,7 +190,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -198,8 +198,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -207,15 +207,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
index aa07d11..b58ef12 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -178,7 +178,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -186,8 +186,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -195,15 +195,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
index e1535ac..e43eb3e 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -210,7 +210,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -218,8 +218,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -227,12 +227,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
index c481125..22dd63a 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -178,7 +178,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -186,8 +186,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -195,15 +195,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
index 5f8686a..cb0a4f4 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -210,7 +210,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -218,8 +218,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -227,12 +227,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
index 609ec98..b0554bc 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
@@ -157,8 +157,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -183,7 +183,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -191,8 +191,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -200,15 +200,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
index 42aeaaa..2d31d96 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -143,7 +143,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -151,8 +151,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -160,15 +160,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
index 367ed89..bc3e045 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,7 +145,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -153,8 +153,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -162,15 +162,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
index 156d22e..083b15f 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
__m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -175,7 +175,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -183,8 +183,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -192,12 +192,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
index 5e401f5..f470c48 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
__m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -177,7 +177,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -185,8 +185,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -194,12 +194,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
index d55b408..c8d66cb 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -143,7 +143,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -151,8 +151,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -160,15 +160,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
index b7e914a..57b96a5 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,7 +145,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -153,8 +153,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -162,15 +162,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
index fbd11c4..148842d 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -175,7 +175,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -183,8 +183,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -192,12 +192,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
index e64c763..6eb9941 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -177,7 +177,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -185,8 +185,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -194,12 +194,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
index 02387e4..a37bb42 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld128.c
@@ -149,8 +149,8 @@
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
const v128_t vacc2x01 = wasm_v32x4_shuffle(vacc2x0123, vsign2x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -166,27 +166,27 @@
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
const v128_t vq31prod2x0123 = wasm_v32x4_shuffle(vprod2x01, vprod2x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
const v128_t vrem2x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod2x0123, vremainder_mask), wasm_i32x4_lt(vq31prod2x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
vacc2x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod2x0123, vshift), wasm_i32x4_gt(vrem2x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vacc22x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc2x0123, vacc2x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc22x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
index 150bbfb..6ed27ee 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-wasmsimd-ld64.c
@@ -145,8 +145,8 @@
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
const v128_t vacc2x01 = wasm_v32x4_shuffle(vacc2x0123, vsign2x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -162,27 +162,27 @@
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
const v128_t vq31prod2x0123 = wasm_v32x4_shuffle(vprod2x01, vprod2x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
const v128_t vrem2x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod2x0123, vremainder_mask), wasm_i32x4_lt(vq31prod2x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
vacc2x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod2x0123, vshift), wasm_i32x4_gt(vrem2x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vacc22x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc2x0123, vacc2x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc22x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
index 7bea2fd..c8245f1 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
@@ -122,8 +122,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -148,7 +148,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -156,8 +156,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -165,15 +165,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
index 5bafe10..560ba40 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
@@ -124,8 +124,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -150,7 +150,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -158,8 +158,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -167,15 +167,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
index 544b4e2..3441a20 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -141,7 +141,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -149,8 +149,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -158,15 +158,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
index d4fb08a..9178d3e 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
__m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -173,7 +173,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -181,8 +181,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -190,12 +190,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
index b9df585..1e5ef35 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -141,7 +141,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -149,8 +149,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -158,15 +158,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
index fd41846..65119a7 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -173,7 +173,7 @@
const __m128i vq31prod1x0123 = _mm_shuffle_epi32(vq31prod1x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -181,8 +181,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -190,12 +190,12 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-wasmsimd.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-wasmsimd.c
index 5b8c51e..adb57cf 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-wasmsimd.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-wasmsimd.c
@@ -145,8 +145,8 @@
const v128_t vacc1x01 = wasm_v32x4_shuffle(vacc1x0123, vsign1x0123, 0, 4, 1, 5);
const v128_t vacc2x01 = wasm_v32x4_shuffle(vacc2x0123, vsign2x0123, 0, 4, 1, 5);
- const v128_t vmultiplier = wasm_v128_load(params->wasmsimd.multiplier);
- const v128_t vrounding = wasm_v128_load(params->wasmsimd.rounding);
+ const v128_t vmultiplier = wasm_v128_load(params->gemmlowp_wasmsimd.multiplier);
+ const v128_t vrounding = wasm_v128_load(params->gemmlowp_wasmsimd.rounding);
const v128_t vprod0x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc0x01, vmultiplier), vrounding);
const v128_t vacc0x23 = wasm_v32x4_shuffle(vacc0x0123, vsign0x0123, 2, 6, 3, 7);
const v128_t vprod1x01 = wasm_i64x2_add(wasm_i64x2_mul(vacc1x01, vmultiplier), vrounding);
@@ -162,27 +162,27 @@
const v128_t vq31prod1x0123 = wasm_v32x4_shuffle(vprod1x01, vprod1x23, 1, 3, 5, 7);
const v128_t vq31prod2x0123 = wasm_v32x4_shuffle(vprod2x01, vprod2x23, 1, 3, 5, 7);
- const v128_t vremainder_mask = wasm_v128_load(params->wasmsimd.remainder_mask);
+ const v128_t vremainder_mask = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_mask);
const v128_t vrem0x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod0x0123, vremainder_mask), wasm_i32x4_lt(vq31prod0x0123, vzero));
const v128_t vrem1x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod1x0123, vremainder_mask), wasm_i32x4_lt(vq31prod1x0123, vzero));
const v128_t vrem2x0123 = wasm_i32x4_add(wasm_v128_and(vq31prod2x0123, vremainder_mask), wasm_i32x4_lt(vq31prod2x0123, vzero));
- const v128_t vthreshold = wasm_v128_load(params->wasmsimd.remainder_threshold);
- const int32_t vshift = params->wasmsimd.shift;
+ const v128_t vthreshold = wasm_v128_load(params->gemmlowp_wasmsimd.remainder_threshold);
+ const int32_t vshift = params->gemmlowp_wasmsimd.shift;
vacc0x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod0x0123, vshift), wasm_i32x4_gt(vrem0x0123, vthreshold));
vacc1x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod1x0123, vshift), wasm_i32x4_gt(vrem1x0123, vthreshold));
vacc2x0123 = wasm_i32x4_sub(wasm_i32x4_shr(vq31prod2x0123, vshift), wasm_i32x4_gt(vrem2x0123, vthreshold));
- const v128_t voutput_zero_point = wasm_v128_load(params->wasmsimd.output_zero_point);
+ const v128_t voutput_zero_point = wasm_v128_load(params->gemmlowp_wasmsimd.output_zero_point);
v128_t vacc01x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc0x0123, vacc1x0123), voutput_zero_point);
v128_t vacc22x0123 = wasm_i16x8_add_saturate(wasm_i16x8_narrow_i32x4(vacc2x0123, vacc2x0123), voutput_zero_point);
v128_t vout = wasm_i8x16_narrow_i16x8(vacc01x0123, vacc22x0123);
- const v128_t voutput_min = wasm_v128_load(params->wasmsimd.output_min);
+ const v128_t voutput_min = wasm_v128_load(params->gemmlowp_wasmsimd.output_min);
vout = wasm_i8x16_max(vout, voutput_min);
- const v128_t voutput_max = wasm_v128_load(params->wasmsimd.output_max);
+ const v128_t voutput_max = wasm_v128_load(params->gemmlowp_wasmsimd.output_max);
vout = wasm_i8x16_min(vout, voutput_max);
if (nc >= 4) {
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
index 995ea8b..7a091ab 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
@@ -120,8 +120,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -146,7 +146,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -154,8 +154,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -163,15 +163,15 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 6126c00..b8f255c 100644
--- a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -237,7 +237,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -245,7 +245,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -261,7 +261,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -277,8 +277,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
index cd20044..9ec08f6 100644
--- a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -236,7 +236,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -244,7 +244,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -260,7 +260,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -276,8 +276,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
index aa3397d..f6608d9 100644
--- a/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/3x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -257,7 +257,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -265,7 +265,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -281,7 +281,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -297,8 +297,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 3dd59a6..6c73b05 100644
--- a/src/qs8-gemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/3x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -234,7 +234,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -242,7 +242,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -258,7 +258,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -273,8 +273,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index e3abc2a..ee3d8fa 100644
--- a/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -286,7 +286,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -294,7 +294,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -310,7 +310,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -326,8 +326,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 428330a..7a1ffad 100644
--- a/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/3x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -186,7 +186,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -194,7 +194,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -210,7 +210,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -226,8 +226,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..60b38a7
--- /dev/null
+++ b/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,212 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
+ const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
+ const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ __m256i vacc2x01 = vacc0x01;
+ __m256i vacc2x23 = vacc0x23;
+ __m256i vacc2x45 = vacc0x45;
+ __m256i vacc2x67 = vacc0x67;
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a2));
+ const __m256i vxa2 = _mm256_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+ vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
+
+ w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+ const __m256i vacc2x0213 = _mm256_hadd_epi32(vacc2x01, vacc2x23);
+ const __m256i vacc2x4657 = _mm256_hadd_epi32(vacc2x45, vacc2x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+ const __m256i vacc2x02461357 = _mm256_hadd_epi32(vacc2x0213, vacc2x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+ __m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+ __m256 vscaled2x01234567 = _mm256_cvtepi32_ps(vacc2x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+ vscaled2x01234567 = _mm256_mul_ps(vscaled2x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+ vacc2x01234567 = _mm256_cvtps_epi32(vscaled2x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+ __m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+ vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+ _mm_storeh_pi((__m64*) c2, _mm_castsi128_ps(vout_lo));
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+ _mm_storeu_si32(c1, vout_hi);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout_lo, 2);
+
+ c0 += 4;
+ c1 += 4;
+ c2 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout_lo, 4);
+
+ c0 += 2;
+ c1 += 2;
+ c2 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
index 68c2e34..b87cc62 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
@@ -133,8 +133,8 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
@@ -159,7 +159,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -167,8 +167,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -176,7 +176,7 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
@@ -185,8 +185,8 @@
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 564e036..85349cb 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -311,7 +311,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -319,7 +319,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -335,7 +335,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -350,8 +350,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
index 38f9282..deb14a7 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -210,7 +210,7 @@
int32x4_t vacc2x4567 = vcombine_s32(vsum2x45, vsum2x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -218,7 +218,7 @@
vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier);
vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -234,7 +234,7 @@
vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift);
vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -249,8 +249,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x8_t vout2x01234567 = vqmovn_s16(vacc2x01234567);
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567 = vmax_s8(vout2x01234567, vget_low_s8(voutput_min));
diff --git a/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
index 6772f39..0a4326c 100644
--- a/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
@@ -129,8 +129,8 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
- const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.rounding);
const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
@@ -155,7 +155,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -163,8 +163,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -172,7 +172,7 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
@@ -181,8 +181,8 @@
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->gemmlowp_avx2.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 249c1a7..5ee36f5 100644
--- a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -446,7 +446,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -464,7 +464,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -500,7 +500,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -530,8 +530,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
index 90c23bc..0e78e76 100644
--- a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -444,7 +444,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -462,7 +462,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -498,7 +498,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -528,8 +528,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
index 000c5eb..a0bba2b 100644
--- a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -523,7 +523,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -541,7 +541,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -577,7 +577,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -607,8 +607,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
index 70da22e..1a7fde3 100644
--- a/src/qs8-gemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/4x16c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -509,7 +509,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -527,7 +527,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -563,7 +563,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -592,8 +592,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index 1f1884f..71a2a88 100644
--- a/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -596,7 +596,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -614,7 +614,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -650,7 +650,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -680,8 +680,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
index 746bcb3..ed74770 100644
--- a/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/4x16c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -358,7 +358,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -376,7 +376,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -412,7 +412,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -442,8 +442,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/4x16c4-minmax-gemmlowp-neondot.c
index c21610b..6b931a0 100644
--- a/src/qs8-gemm/gen/4x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/4x16c4-minmax-gemmlowp-neondot.c
@@ -169,25 +169,25 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->neon.multiplier);
- const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->neon.multiplier);
- const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->neon.multiplier);
- const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->neon.multiplier);
- const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -223,7 +223,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -253,8 +253,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
index 5439a53..8ff8bbe 100644
--- a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
+++ b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-avx512skx.c
@@ -61,14 +61,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
diff --git a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
index f03e0e8..7ce34e0 100644
--- a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -684,7 +684,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -702,7 +702,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -738,7 +738,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -767,8 +767,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
index c5dea1e..2fb0196 100644
--- a/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/4x16c8-minmax-gemmlowp-neon-mull-padal.c
@@ -445,7 +445,7 @@
int32x4_t vacc3xCDEF = vcombine_s32(vsum3xCD, vsum3xEF );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -463,7 +463,7 @@
vacc3x89AB = vqrdmulhq_s32(vacc3x89AB, vmultiplier);
vacc3xCDEF = vqrdmulhq_s32(vacc3xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -499,7 +499,7 @@
vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_shift);
vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -528,8 +528,8 @@
int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/4x2-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/4x2-minmax-gemmlowp-scalar.c
index 9679d29..608db29 100644
--- a/src/qs8-gemm/gen/4x2-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/4x2-minmax-gemmlowp-scalar.c
@@ -85,7 +85,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct1x0 = (int64_t) vacc1x0 * (int64_t) vmultiplier;
@@ -105,7 +105,7 @@
const int32_t vq31product3x0 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x0 + vq31rounding) >> 31);
const int32_t vq31product3x1 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x1 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder1x0 = (vq31product1x0 & vremainder_mask) - (int32_t) (vq31product1x0 < 0);
@@ -115,8 +115,8 @@
const int32_t vremainder3x0 = (vq31product3x0 & vremainder_mask) - (int32_t) (vq31product3x0 < 0);
const int32_t vremainder3x1 = (vq31product3x1 & vremainder_mask) - (int32_t) (vq31product3x1 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout1x0 = asr_s32(vq31product1x0, vshift) + (int32_t) (vremainder1x0 > vremainder_threshold);
@@ -126,7 +126,7 @@
int32_t vout3x0 = asr_s32(vq31product3x0, vshift) + (int32_t) (vremainder3x0 > vremainder_threshold);
int32_t vout3x1 = asr_s32(vq31product3x1, vshift) + (int32_t) (vremainder3x1 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout1x0 = math_max_s32(vout1x0, vout_min);
@@ -136,7 +136,7 @@
vout3x0 = math_max_s32(vout3x0, vout_min);
vout3x1 = math_max_s32(vout3x1, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout1x0 = math_min_s32(vout1x0, vout_max);
@@ -146,7 +146,7 @@
vout3x0 = math_min_s32(vout3x0, vout_max);
vout3x1 = math_min_s32(vout3x1, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout1x0 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/4x4-minmax-gemmlowp-scalar.c b/src/qs8-gemm/gen/4x4-minmax-gemmlowp-scalar.c
index 16419b8..74c469f 100644
--- a/src/qs8-gemm/gen/4x4-minmax-gemmlowp-scalar.c
+++ b/src/qs8-gemm/gen/4x4-minmax-gemmlowp-scalar.c
@@ -103,7 +103,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
const int64_t vproduct0x0 = (int64_t) vacc0x0 * (int64_t) vmultiplier;
const int64_t vproduct0x1 = (int64_t) vacc0x1 * (int64_t) vmultiplier;
const int64_t vproduct0x2 = (int64_t) vacc0x2 * (int64_t) vmultiplier;
@@ -139,7 +139,7 @@
const int32_t vq31product3x2 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x2 + vq31rounding) >> 31);
const int32_t vq31product3x3 = (int32_t) (uint32_t) ((uint64_t) (vproduct3x3 + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
const int32_t vremainder0x0 = (vq31product0x0 & vremainder_mask) - (int32_t) (vq31product0x0 < 0);
const int32_t vremainder0x1 = (vq31product0x1 & vremainder_mask) - (int32_t) (vq31product0x1 < 0);
const int32_t vremainder0x2 = (vq31product0x2 & vremainder_mask) - (int32_t) (vq31product0x2 < 0);
@@ -157,8 +157,8 @@
const int32_t vremainder3x2 = (vq31product3x2 & vremainder_mask) - (int32_t) (vq31product3x2 < 0);
const int32_t vremainder3x3 = (vq31product3x3 & vremainder_mask) - (int32_t) (vq31product3x3 < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
int32_t vout0x0 = asr_s32(vq31product0x0, vshift) + (int32_t) (vremainder0x0 > vremainder_threshold);
int32_t vout0x1 = asr_s32(vq31product0x1, vshift) + (int32_t) (vremainder0x1 > vremainder_threshold);
int32_t vout0x2 = asr_s32(vq31product0x2, vshift) + (int32_t) (vremainder0x2 > vremainder_threshold);
@@ -176,7 +176,7 @@
int32_t vout3x2 = asr_s32(vq31product3x2, vshift) + (int32_t) (vremainder3x2 > vremainder_threshold);
int32_t vout3x3 = asr_s32(vq31product3x3, vshift) + (int32_t) (vremainder3x3 > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
vout0x0 = math_max_s32(vout0x0, vout_min);
vout0x1 = math_max_s32(vout0x1, vout_min);
vout0x2 = math_max_s32(vout0x2, vout_min);
@@ -194,7 +194,7 @@
vout3x2 = math_max_s32(vout3x2, vout_min);
vout3x3 = math_max_s32(vout3x3, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
vout0x0 = math_min_s32(vout0x0, vout_max);
vout0x1 = math_min_s32(vout0x1, vout_max);
vout0x2 = math_min_s32(vout0x2, vout_max);
@@ -212,7 +212,7 @@
vout3x2 = math_min_s32(vout3x2, vout_max);
vout3x3 = math_min_s32(vout3x3, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
vout0x0 += voutput_zero_point;
vout0x1 += voutput_zero_point;
vout0x2 += voutput_zero_point;
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
index b0cec37..6ee3c62 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,15 +239,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
index a3db995..939d732 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,15 +239,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
index 17f7643..df5ede3 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -258,7 +258,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -268,8 +268,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -279,12 +279,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
index d2f821c..9882976 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -258,7 +258,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -268,8 +268,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -279,12 +279,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
index 2d53868..a7ea9d9 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,15 +239,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
index dafa8dc..64fb13f 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,15 +239,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
index 9c8cf70..739e36b 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -258,7 +258,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -268,8 +268,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -279,12 +279,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
index 1225935..fb97abd 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -258,7 +258,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -268,8 +268,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -279,12 +279,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
index 10bf278..4a2cf7b 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
@@ -191,8 +191,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -223,7 +223,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -233,8 +233,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -244,15 +244,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
index 4e2b48a..0af6be9 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
@@ -191,8 +191,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -223,7 +223,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -233,8 +233,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -244,15 +244,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
index 06a057a..de0f304 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -211,7 +211,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -221,8 +221,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -232,15 +232,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
index 127c98b..ed56800 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -251,7 +251,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -261,8 +261,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -272,12 +272,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
index 93f66b1..29df1a0 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -211,7 +211,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -221,8 +221,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -232,15 +232,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
index 357bec9..c52fd01 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.rounding);
const __m128i vnmask0x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc0x0123);
const __m128i vnmask1x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc1x0123);
@@ -251,7 +251,7 @@
const __m128i vq31prod2x0123 = _mm_shuffle_epi32(vq31prod2x0213, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod3x0123 = _mm_shuffle_epi32(vq31prod3x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -261,8 +261,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -272,12 +272,12 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
index 61952bd..b4a5242 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
@@ -184,8 +184,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -216,7 +216,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -226,8 +226,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->gemmlowp_sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -237,15 +237,15 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 36b85b2..9b6ed23 100644
--- a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -279,7 +279,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -289,7 +289,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -309,7 +309,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -327,8 +327,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
index fa00c3a..de22292 100644
--- a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -278,7 +278,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -288,7 +288,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -308,7 +308,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -326,8 +326,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
index c53e78c..2c78c76 100644
--- a/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
+++ b/src/qs8-gemm/gen/4x8-minmax-gemmlowp-neon-mull-addw-dup.c
@@ -312,7 +312,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -322,7 +322,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -342,7 +342,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -360,8 +360,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
index 4351ac3..1ee332c 100644
--- a/src/qs8-gemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/4x8c16-minmax-gemmlowp-neon-mlal-padal.c
@@ -293,7 +293,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -303,7 +303,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -323,7 +323,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -340,8 +340,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c b/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
index acf3d2e..42b9632 100644
--- a/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
+++ b/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mlal-padal-dup.c
@@ -350,7 +350,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -360,7 +360,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -380,7 +380,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -398,8 +398,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c b/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
index d640aef..c7acc44 100644
--- a/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
+++ b/src/qs8-gemm/gen/4x8c2-minmax-gemmlowp-neon-mull-padal-dup.c
@@ -224,7 +224,7 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -234,7 +234,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -254,7 +254,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -272,8 +272,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/4x8c4-minmax-gemmlowp-neondot.c
index 949f81b..266a756 100644
--- a/src/qs8-gemm/gen/4x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/4x8c4-minmax-gemmlowp-neondot.c
@@ -131,17 +131,17 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -161,7 +161,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -179,8 +179,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c b/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
index 5d4c337..b4ad38c 100644
--- a/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mlal-padal.c
@@ -388,7 +388,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -398,7 +398,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -418,7 +418,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -435,8 +435,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c b/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
index 81a0a81..7042d13 100644
--- a/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
+++ b/src/qs8-gemm/gen/4x8c8-minmax-gemmlowp-neon-mull-padal.c
@@ -261,7 +261,7 @@
int32x4_t vacc3x4567 = vcombine_s32(vsum3x45, vsum3x67 );
#endif
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -271,7 +271,7 @@
vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier);
vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -291,7 +291,7 @@
vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift);
vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -308,8 +308,8 @@
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 3a9b99b..5fd4633 100644
--- a/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -594,7 +594,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -620,7 +620,7 @@
vacc5x89AB = vqrdmulhq_s32(vacc5x89AB, vmultiplier);
vacc5xCDEF = vqrdmulhq_s32(vacc5xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -672,7 +672,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -714,8 +714,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
index b7bb61b..2812abd 100644
--- a/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/6x16-minmax-gemmlowp-neon-mlal-lane.c
@@ -592,7 +592,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc0x89AB = vqrdmulhq_s32(vacc0x89AB, vmultiplier);
@@ -618,7 +618,7 @@
vacc5x89AB = vqrdmulhq_s32(vacc5x89AB, vmultiplier);
vacc5xCDEF = vqrdmulhq_s32(vacc5xCDEF, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -670,7 +670,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -712,8 +712,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/6x16c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/6x16c4-minmax-gemmlowp-neondot.c
index 8eb91a7..abf5913 100644
--- a/src/qs8-gemm/gen/6x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/6x16c4-minmax-gemmlowp-neondot.c
@@ -217,33 +217,33 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->neon.multiplier);
- const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->neon.multiplier);
- const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->neon.multiplier);
- const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->neon.multiplier);
- const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->neon.multiplier);
- const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier);
- const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier);
- const int32x4_t vproduct4x89AB = vqrdmulhq_n_s32(vacc4x89AB, params->neon.multiplier);
- const int32x4_t vproduct4xCDEF = vqrdmulhq_n_s32(vacc4xCDEF, params->neon.multiplier);
- const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier);
- const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier);
- const int32x4_t vproduct5x89AB = vqrdmulhq_n_s32(vacc5x89AB, params->neon.multiplier);
- const int32x4_t vproduct5xCDEF = vqrdmulhq_n_s32(vacc5xCDEF, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x89AB = vqrdmulhq_n_s32(vacc4x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4xCDEF = vqrdmulhq_n_s32(vacc4xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x89AB = vqrdmulhq_n_s32(vacc5x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5xCDEF = vqrdmulhq_n_s32(vacc5xCDEF, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -295,7 +295,7 @@
vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_shift);
vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -337,8 +337,8 @@
int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c b/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
index 2bb0e53..b08dfd9 100644
--- a/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
+++ b/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane-prfm.c
@@ -363,7 +363,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -377,7 +377,7 @@
vacc5x0123 = vqrdmulhq_s32(vacc5x0123, vmultiplier);
vacc5x4567 = vqrdmulhq_s32(vacc5x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -405,7 +405,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -429,8 +429,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c b/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
index d36d452..2336529 100644
--- a/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/6x8-minmax-gemmlowp-neon-mlal-lane.c
@@ -362,7 +362,7 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier);
vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier);
vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier);
@@ -376,7 +376,7 @@
vacc5x0123 = vqrdmulhq_s32(vacc5x0123, vmultiplier);
vacc5x4567 = vqrdmulhq_s32(vacc5x4567, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -404,7 +404,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -428,8 +428,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/6x8c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/6x8c4-minmax-gemmlowp-neondot.c
index 230bbff..6a54838 100644
--- a/src/qs8-gemm/gen/6x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/6x8c4-minmax-gemmlowp-neondot.c
@@ -163,21 +163,21 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier);
- const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier);
- const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier);
- const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -205,7 +205,7 @@
vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift);
vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -229,8 +229,8 @@
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/gen/8x16c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/8x16c4-minmax-gemmlowp-neondot.c
index da350ec..78f241d 100644
--- a/src/qs8-gemm/gen/8x16c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/8x16c4-minmax-gemmlowp-neondot.c
@@ -265,41 +265,41 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->neon.multiplier);
- const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->neon.multiplier);
- const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->neon.multiplier);
- const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->neon.multiplier);
- const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->neon.multiplier);
- const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier);
- const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier);
- const int32x4_t vproduct4x89AB = vqrdmulhq_n_s32(vacc4x89AB, params->neon.multiplier);
- const int32x4_t vproduct4xCDEF = vqrdmulhq_n_s32(vacc4xCDEF, params->neon.multiplier);
- const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier);
- const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier);
- const int32x4_t vproduct5x89AB = vqrdmulhq_n_s32(vacc5x89AB, params->neon.multiplier);
- const int32x4_t vproduct5xCDEF = vqrdmulhq_n_s32(vacc5xCDEF, params->neon.multiplier);
- const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->neon.multiplier);
- const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->neon.multiplier);
- const int32x4_t vproduct6x89AB = vqrdmulhq_n_s32(vacc6x89AB, params->neon.multiplier);
- const int32x4_t vproduct6xCDEF = vqrdmulhq_n_s32(vacc6xCDEF, params->neon.multiplier);
- const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->neon.multiplier);
- const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->neon.multiplier);
- const int32x4_t vproduct7x89AB = vqrdmulhq_n_s32(vacc7x89AB, params->neon.multiplier);
- const int32x4_t vproduct7xCDEF = vqrdmulhq_n_s32(vacc7xCDEF, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x89AB = vqrdmulhq_n_s32(vacc0x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0xCDEF = vqrdmulhq_n_s32(vacc0xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x89AB = vqrdmulhq_n_s32(vacc1x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1xCDEF = vqrdmulhq_n_s32(vacc1xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x89AB = vqrdmulhq_n_s32(vacc2x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2xCDEF = vqrdmulhq_n_s32(vacc2xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x89AB = vqrdmulhq_n_s32(vacc3x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3xCDEF = vqrdmulhq_n_s32(vacc3xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x89AB = vqrdmulhq_n_s32(vacc4x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4xCDEF = vqrdmulhq_n_s32(vacc4xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x89AB = vqrdmulhq_n_s32(vacc5x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5xCDEF = vqrdmulhq_n_s32(vacc5xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x89AB = vqrdmulhq_n_s32(vacc6x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6xCDEF = vqrdmulhq_n_s32(vacc6xCDEF, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x89AB = vqrdmulhq_n_s32(vacc7x89AB, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7xCDEF = vqrdmulhq_n_s32(vacc7xCDEF, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -367,7 +367,7 @@
vacc7x89AB = vrshlq_s32(vacc7x89AB, vright_shift);
vacc7xCDEF = vrshlq_s32(vacc7xCDEF, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
@@ -421,8 +421,8 @@
int8x16_t vout6x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc6x89ABCDEF));
int8x16_t vout7x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc7x01234567), vqmovn_s16(vacc7x89ABCDEF));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
diff --git a/src/qs8-gemm/gen/8x8c4-minmax-gemmlowp-neondot.c b/src/qs8-gemm/gen/8x8c4-minmax-gemmlowp-neondot.c
index 1c9501d..a01922e 100644
--- a/src/qs8-gemm/gen/8x8c4-minmax-gemmlowp-neondot.c
+++ b/src/qs8-gemm/gen/8x8c4-minmax-gemmlowp-neondot.c
@@ -195,25 +195,25 @@
}
// Post-accumulation work
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
- const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier);
- const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier);
- const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier);
- const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier);
- const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier);
- const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier);
- const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier);
- const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier);
- const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier);
- const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier);
- const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier);
- const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier);
- const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->neon.multiplier);
- const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->neon.multiplier);
- const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->neon.multiplier);
- const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->neon.multiplier);
+ const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->gemmlowp_neon.multiplier);
+ const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->gemmlowp_neon.multiplier);
vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31);
vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31);
@@ -249,7 +249,7 @@
vacc7x0123 = vrshlq_s32(vacc7x0123, vright_shift);
vacc7x4567 = vrshlq_s32(vacc7x4567, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
@@ -279,8 +279,8 @@
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
int8x16_t vout6x01234567_7x01234567 = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc7x01234567));
#endif
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
diff --git a/src/qs8-gemm/neon-mlal-lane.c.in b/src/qs8-gemm/neon-mlal-lane.c.in
index 6db2cae..e1a2829 100644
--- a/src/qs8-gemm/neon-mlal-lane.c.in
+++ b/src/qs8-gemm/neon-mlal-lane.c.in
@@ -174,12 +174,12 @@
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -189,7 +189,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -218,11 +218,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/neon-mull-addw-dup.c.in b/src/qs8-gemm/neon-mull-addw-dup.c.in
index 42e01dd..8cbf8aa 100644
--- a/src/qs8-gemm/neon-mull-addw-dup.c.in
+++ b/src/qs8-gemm/neon-mull-addw-dup.c.in
@@ -158,12 +158,12 @@
}
}
}
- const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->gemmlowp_neon.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
- const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift);
+ const int32x4_t vright_shift = vld1q_dup_s32(¶ms->gemmlowp_neon.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
@@ -173,7 +173,7 @@
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
- const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->gemmlowp_neon.output_zero_point);
#if XNN_ARCH_ARM64
$for M in range(MR):
$for N in range(0, NR, 8):
@@ -202,11 +202,11 @@
int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]});
#endif
$if NR == 8 and MR == 1:
- const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
- const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->gemmlowp_neon.output_max);
$else:
- const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
- const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->gemmlowp_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->gemmlowp_neon.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
diff --git a/src/qs8-gemm/scalar.c.in b/src/qs8-gemm/scalar.c.in
index 4b813a2..4d8f133 100644
--- a/src/qs8-gemm/scalar.c.in
+++ b/src/qs8-gemm/scalar.c.in
@@ -71,7 +71,7 @@
k -= sizeof(int8_t);
} while (k != 0);
- const int32_t vmultiplier = params->scalar.multiplier;
+ const int32_t vmultiplier = params->gemmlowp_scalar.multiplier;
$for M in range(MR):
$for N in range(NR):
const int64_t vproduct${M}x${N} = (int64_t) vacc${M}x${N} * (int64_t) vmultiplier;
@@ -81,28 +81,28 @@
$for N in range(NR):
const int32_t vq31product${M}x${N} = (int32_t) (uint32_t) ((uint64_t) (vproduct${M}x${N} + vq31rounding) >> 31);
- const int32_t vremainder_mask = params->scalar.remainder_mask;
+ const int32_t vremainder_mask = params->gemmlowp_scalar.remainder_mask;
$for M in range(MR):
$for N in range(NR):
const int32_t vremainder${M}x${N} = (vq31product${M}x${N} & vremainder_mask) - (int32_t) (vq31product${M}x${N} < 0);
- const uint32_t vshift = params->scalar.shift;
- const int32_t vremainder_threshold = params->scalar.remainder_threshold;
+ const uint32_t vshift = params->gemmlowp_scalar.shift;
+ const int32_t vremainder_threshold = params->gemmlowp_scalar.remainder_threshold;
$for M in range(MR):
$for N in range(NR):
int32_t vout${M}x${N} = asr_s32(vq31product${M}x${N}, vshift) + (int32_t) (vremainder${M}x${N} > vremainder_threshold);
- const int32_t vout_min = params->scalar.output_min_less_zero_point;
+ const int32_t vout_min = params->gemmlowp_scalar.output_min_less_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} = math_max_s32(vout${M}x${N}, vout_min);
- const int32_t vout_max = params->scalar.output_max_less_zero_point;
+ const int32_t vout_max = params->gemmlowp_scalar.output_max_less_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} = math_min_s32(vout${M}x${N}, vout_max);
- const int32_t voutput_zero_point = params->scalar.output_zero_point;
+ const int32_t voutput_zero_point = params->gemmlowp_scalar.output_zero_point;
$for M in range(MR):
$for N in range(NR):
vout${M}x${N} += voutput_zero_point;